Drupal investigation

migrate_tools.drush.inc 18KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499
  1. <?php
  2. /**
  3. * @file
  4. * Command-line tools to aid performing and developing migrations.
  5. */
  6. use Drupal\Component\Utility\Unicode;
  7. use Drupal\migrate\Plugin\MigrationInterface;
  8. use Drupal\migrate_tools\MigrateExecutable;
  9. use Drupal\migrate_tools\DrushLogMigrateMessage;
  10. use Drupal\Core\Datetime\DateFormatter;
  11. use Drupal\migrate_plus\Entity\MigrationGroup;
  12. /**
  13. * Implements hook_drush_command().
  14. */
  15. function migrate_tools_drush_command() {
  16. $items['migrate-status'] = [
  17. 'description' => 'List all migrations with current status.',
  18. 'options' => [
  19. 'group' => 'A comma-separated list of migration groups to list',
  20. 'tag' => 'Name of the migration tag to list',
  21. 'names-only' => 'Only return names, not all the details (faster)',
  22. ],
  23. 'arguments' => [
  24. 'migration' => 'Restrict to a comma-separated list of migrations. Optional',
  25. ],
  26. 'examples' => [
  27. 'migrate-status' => 'Retrieve status for all migrations',
  28. 'migrate-status --group=beer' => 'Retrieve status for all migrations in a given group',
  29. 'migrate-status --tag=user' => 'Retrieve status for all migrations with a given tag',
  30. 'migrate-status --group=beer --tag=user' => 'Retrieve status for all migrations in the beer group and with the user tag',
  31. 'migrate-status beer_term,beer_node' => 'Retrieve status for specific migrations',
  32. ],
  33. 'drupal dependencies' => ['migrate_tools'],
  34. 'aliases' => ['ms'],
  35. ];
  36. $items['migrate-import'] = [
  37. 'description' => 'Perform one or more migration processes.',
  38. 'options' => [
  39. 'all' => 'Process all migrations.',
  40. 'group' => 'A comma-separated list of migration groups to import',
  41. 'tag' => 'Name of the migration tag to import',
  42. 'limit' => 'Limit on the number of items to process in each migration',
  43. 'feedback' => 'Frequency of progress messages, in items processed',
  44. 'idlist' => 'Comma-separated list of IDs to import',
  45. 'update' => ' In addition to processing unprocessed items from the source, update previously-imported items with the current data',
  46. 'force' => 'Force an operation to run, even if all dependencies are not satisfied',
  47. 'execute-dependencies' => 'Execute all dependent migrations first.',
  48. ],
  49. 'arguments' => [
  50. 'migration' => 'ID of migration(s) to import. Delimit multiple using commas.',
  51. ],
  52. 'examples' => [
  53. 'migrate-import --all' => 'Perform all migrations',
  54. 'migrate-import --group=beer' => 'Import all migrations in the beer group',
  55. 'migrate-import --tag=user' => 'Import all migrations with the user tag',
  56. 'migrate-import --group=beer --tag=user' => 'Import all migrations in the beer group and with the user tag',
  57. 'migrate-import beer_term,beer_node' => 'Import new terms and nodes',
  58. 'migrate-import beer_user --limit=2' => 'Import no more than 2 users',
  59. 'migrate-import beer_user --idlist=5' => 'Import the user record with source ID 5',
  60. ],
  61. 'drupal dependencies' => ['migrate_tools'],
  62. 'aliases' => ['mi'],
  63. ];
  64. $items['migrate-rollback'] = array(
  65. 'description' => 'Rollback one or more migrations.',
  66. 'options' => array(
  67. 'all' => 'Process all migrations.',
  68. 'group' => 'A comma-separated list of migration groups to rollback',
  69. 'tag' => 'ID of the migration tag to rollback',
  70. 'feedback' => 'Frequency of progress messages, in items processed',
  71. ),
  72. 'arguments' => array(
  73. 'migration' => 'Name of migration(s) to rollback. Delimit multiple using commas.',
  74. ),
  75. 'examples' => array(
  76. 'migrate-rollback --all' => 'Perform all migrations',
  77. 'migrate-rollback --group=beer' => 'Rollback all migrations in the beer group',
  78. 'migrate-rollback --tag=user' => 'Rollback all migrations with the user tag',
  79. 'migrate-rollback --group=beer --tag=user' => 'Rollback all migrations in the beer group and with the user tag',
  80. 'migrate-rollback beer_term,beer_node' => 'Rollback imported terms and nodes',
  81. ),
  82. 'drupal dependencies' => array('migrate_tools'),
  83. 'aliases' => array('mr'),
  84. );
  85. $items['migrate-stop'] = [
  86. 'description' => 'Stop an active migration operation.',
  87. 'arguments' => [
  88. 'migration' => 'ID of migration to stop',
  89. ],
  90. 'drupal dependencies' => ['migrate_tools'],
  91. 'aliases' => ['mst'],
  92. ];
  93. $items['migrate-reset-status'] = [
  94. 'description' => 'Reset a active migration\'s status to idle.',
  95. 'arguments' => [
  96. 'migration' => 'ID of migration to reset',
  97. ],
  98. 'drupal dependencies' => ['migrate_tools'],
  99. 'aliases' => ['mrs'],
  100. ];
  101. $items['migrate-messages'] = [
  102. 'description' => 'View any messages associated with a migration.',
  103. 'arguments' => [
  104. 'migration' => 'ID of the migration',
  105. ],
  106. 'options' => [
  107. 'csv' => 'Export messages as a CSV'
  108. ],
  109. 'examples' => [
  110. 'migrate-messages MyNode' => 'Show all messages for the MyNode migration',
  111. ],
  112. 'drupal dependencies' => ['migrate_tools'],
  113. 'aliases' => ['mmsg'],
  114. ];
  115. $items['migrate-fields-source'] = [
  116. 'description' => 'List the fields available for mapping in a source.',
  117. 'arguments' => [
  118. 'migration' => 'ID of the migration',
  119. ],
  120. 'examples' => [
  121. 'migrate-fields-source my_node' => 'List fields for the source in the my_node migration',
  122. ],
  123. 'drupal dependencies' => ['migrate_tools'],
  124. 'aliases' => ['mfs'],
  125. ];
  126. return $items;
  127. }
  128. /**
  129. * @param string $migration_names
  130. */
  131. function drush_migrate_tools_migrate_status($migration_names = '') {
  132. $names_only = drush_get_option('names-only');
  133. $migrations = drush_migrate_tools_migration_list($migration_names);
  134. $table = [];
  135. // Take it one group at a time, listing the migrations within each group.
  136. foreach ($migrations as $group_id => $migration_list) {
  137. $group = MigrationGroup::load($group_id);
  138. $group_name = !empty($group) ? "{$group->label()} ({$group->id()})" : $group_id;
  139. if ($names_only) {
  140. $table[] = [
  141. dt('Group: @name', array('@name' => $group_name))
  142. ];
  143. }
  144. else {
  145. $table[] = [
  146. dt('Group: @name', array('@name' => $group_name)),
  147. dt('Status'),
  148. dt('Total'),
  149. dt('Imported'),
  150. dt('Unprocessed'),
  151. dt('Last imported'),
  152. ];
  153. }
  154. foreach ($migration_list as $migration_id => $migration) {
  155. try {
  156. $map = $migration->getIdMap();
  157. $imported = $map->importedCount();
  158. $source_plugin = $migration->getSourcePlugin();
  159. }
  160. catch (Exception $e) {
  161. drush_log(dt('Failure retrieving information on @migration: @message',
  162. ['@migration' => $migration_id, '@message' => $e->getMessage()]));
  163. continue;
  164. }
  165. try {
  166. $source_rows = $source_plugin->count();
  167. // -1 indicates uncountable sources.
  168. if ($source_rows == -1) {
  169. $source_rows = dt('N/A');
  170. $unprocessed = dt('N/A');
  171. }
  172. else {
  173. $unprocessed = $source_rows - $map->processedCount();
  174. }
  175. }
  176. catch (Exception $e) {
  177. drush_print($e->getMessage());
  178. drush_log(dt('Could not retrieve source count from @migration: @message',
  179. ['@migration' => $migration_id, '@message' => $e->getMessage()]));
  180. $source_rows = dt('N/A');
  181. $unprocessed = dt('N/A');
  182. }
  183. if ($names_only) {
  184. $table[] = [$migration_id];
  185. }
  186. else {
  187. $status = $migration->getStatusLabel();
  188. $migrate_last_imported_store = \Drupal::keyValue('migrate_last_imported');
  189. $last_imported = $migrate_last_imported_store->get($migration->id(), FALSE);
  190. if ($last_imported) {
  191. /** @var DateFormatter $date_formatter */
  192. $date_formatter = \Drupal::service('date.formatter');
  193. $last_imported = $date_formatter->format($last_imported / 1000,
  194. 'custom', 'Y-m-d H:i:s');
  195. }
  196. else {
  197. $last_imported = '';
  198. }
  199. $table[] = [$migration_id, $status, $source_rows, $imported, $unprocessed, $last_imported];
  200. }
  201. }
  202. }
  203. drush_print_table($table);
  204. }
  205. /**
  206. * @param string $migration_names
  207. */
  208. function drush_migrate_tools_migrate_import($migration_names = '') {
  209. $group_names = drush_get_option('group');
  210. $tag_names = drush_get_option('tag');
  211. $all = drush_get_option('all');
  212. $options = [];
  213. if (!$all && !$group_names && !$migration_names && !$tag_names) {
  214. drush_set_error('MIGRATE_ERROR', dt('You must specify --all, --group, --tag or one or more migration names separated by commas'));
  215. return;
  216. }
  217. foreach (['limit', 'feedback', 'idlist', 'update', 'force'] as $option) {
  218. if (drush_get_option($option)) {
  219. $options[$option] = drush_get_option($option);
  220. }
  221. }
  222. $migrations = drush_migrate_tools_migration_list($migration_names);
  223. if (empty($migrations)) {
  224. drush_log(dt('No migrations found.'), 'error');
  225. }
  226. // Take it one group at a time, importing the migrations within each group.
  227. foreach ($migrations as $group_id => $migration_list) {
  228. array_walk($migration_list, '_drush_migrate_tools_execute_migration', $options);
  229. }
  230. }
  231. /**
  232. * Executes a single migration. If the --execute-dependencies option was given,
  233. * the migration's dependencies will also be executed first.
  234. *
  235. * @param \Drupal\migrate\Plugin\MigrationInterface $migration
  236. * The migration to execute.
  237. * @param string $migration_id
  238. * The migration ID (not used, just an artifact of array_walk()).
  239. * @param array $options
  240. * Additional options for the migration.
  241. */
  242. function _drush_migrate_tools_execute_migration(MigrationInterface $migration, $migration_id, array $options = []) {
  243. $log = new DrushLogMigrateMessage();
  244. if (drush_get_option('execute-dependencies')) {
  245. if ($required_IDS = $migration->get('requirements')) {
  246. $manager = \Drupal::service('plugin.manager.config_entity_migration');
  247. $required_migrations = $manager->createInstances($required_IDS);
  248. $dependency_options = array_merge($options, ['is_dependency' => TRUE]);
  249. array_walk($required_migrations, __FUNCTION__, $dependency_options);
  250. }
  251. }
  252. if (!empty($options['force'])) {
  253. $migration->set('requirements', []);
  254. }
  255. if (!empty($options['update'])) {
  256. $migration->getIdMap()->prepareUpdate();
  257. }
  258. $executable = new MigrateExecutable($migration, $log, $options);
  259. // drush_op() provides --simulate support
  260. drush_op(array($executable, 'import'));
  261. }
  262. /**
  263. * @param string $migration_names
  264. */
  265. function drush_migrate_tools_migrate_rollback($migration_names = '') {
  266. $group_names = drush_get_option('group');
  267. $tag_names = drush_get_option('tag');
  268. $all = drush_get_option('all');
  269. $options = [];
  270. if (!$all && !$group_names && !$migration_names && !$tag_names) {
  271. drush_set_error('MIGRATE_ERROR', dt('You must specify --all, --group, --tag, or one or more migration names separated by commas'));
  272. return;
  273. }
  274. if (drush_get_option('feedback')) {
  275. $options['feedback'] = drush_get_option('feedback');
  276. }
  277. $log = new DrushLogMigrateMessage();
  278. $migrations = drush_migrate_tools_migration_list($migration_names);
  279. if (empty($migrations)) {
  280. drush_log(dt('No migrations found.'), 'error');
  281. }
  282. // Take it one group at a time, rolling back the migrations within each group.
  283. foreach ($migrations as $group_id => $migration_list) {
  284. // Roll back in reverse order.
  285. $migration_list = array_reverse($migration_list);
  286. foreach ($migration_list as $migration_id => $migration) {
  287. $executable = new MigrateExecutable($migration, $log, $options);
  288. // drush_op() provides --simulate support.
  289. drush_op(array($executable, 'rollback'));
  290. }
  291. }
  292. }
  293. /**
  294. * @param string $migration_id
  295. */
  296. function drush_migrate_tools_migrate_stop($migration_id = '') {
  297. /** @var MigrationInterface $migration */
  298. $migration = \Drupal::service('plugin.manager.migration')->createInstance($migration_id);
  299. if ($migration) {
  300. $status = $migration->getStatus();
  301. switch ($status) {
  302. case MigrationInterface::STATUS_IDLE:
  303. drush_log(dt('Migration @id is idle', ['@id' => $migration_id]), 'warning');
  304. break;
  305. case MigrationInterface::STATUS_DISABLED:
  306. drush_log(dt('Migration @id is disabled', ['@id' => $migration_id]), 'warning');
  307. break;
  308. case MigrationInterface::STATUS_STOPPING:
  309. drush_log(dt('Migration @id is already stopping', ['@id' => $migration_id]), 'warning');
  310. break;
  311. default:
  312. $migration->interruptMigration(MigrationInterface::RESULT_STOPPED);
  313. drush_log(dt('Migration @id requested to stop', ['@id' => $migration_id]), 'success');
  314. break;
  315. }
  316. }
  317. else {
  318. drush_log(dt('Migration @id does not exist', ['@id' => $migration_id]), 'error');
  319. }
  320. }
  321. /**
  322. * @param string $migration_id
  323. */
  324. function drush_migrate_tools_migrate_reset_status($migration_id = '') {
  325. /** @var MigrationInterface $migration */
  326. $migration = \Drupal::service('plugin.manager.migration')->createInstance($migration_id);
  327. if ($migration) {
  328. $status = $migration->getStatus();
  329. if ($status == MigrationInterface::STATUS_IDLE) {
  330. drush_log(dt('Migration @id is already Idle', ['@id' => $migration_id]), 'warning');
  331. }
  332. else {
  333. $migration->setStatus(MigrationInterface::STATUS_IDLE);
  334. drush_log(dt('Migration @id reset to Idle', ['@id' => $migration_id]), 'status');
  335. }
  336. }
  337. else {
  338. drush_log(dt('Migration @id does not exist', ['@id' => $migration_id]), 'error');
  339. }
  340. }
  341. /**
  342. * @param string $migration_id
  343. */
  344. function drush_migrate_tools_migrate_messages($migration_id) {
  345. /** @var MigrationInterface $migration */
  346. $migration = \Drupal::service('plugin.manager.migration')->createInstance($migration_id);
  347. if ($migration) {
  348. $map = $migration->getIdMap();
  349. $first = TRUE;
  350. $table = [];
  351. foreach ($map->getMessageIterator() as $row) {
  352. unset($row->msgid);
  353. if ($first) {
  354. // @todo: Ideally, replace sourceid* with source key names. Or, should
  355. // getMessageIterator() do that?
  356. foreach ($row as $column => $value) {
  357. $table[0][] = $column;
  358. }
  359. $first = FALSE;
  360. }
  361. $table[] = (array)$row;
  362. }
  363. if (empty($table)) {
  364. drush_log(dt('No messages for this migration'), 'status');
  365. }
  366. else {
  367. if (drush_get_option('csv')) {
  368. foreach ($table as $row) {
  369. fputcsv(STDOUT, $row);
  370. }
  371. }
  372. else {
  373. $widths = [];
  374. foreach ($table[0] as $header) {
  375. $widths[] = strlen($header) + 1;
  376. }
  377. drush_print_table($table, TRUE, $widths);
  378. }
  379. }
  380. }
  381. else {
  382. drush_log(dt('Migration @id does not exist', ['@id' => $migration_id]), 'error');
  383. }
  384. }
  385. /**
  386. * @param string $migration_id
  387. */
  388. function drush_migrate_tools_migrate_fields_source($migration_id) {
  389. /** @var MigrationInterface $migration */
  390. $migration = \Drupal::service('plugin.manager.migration')->createInstance($migration_id);
  391. if ($migration) {
  392. $source = $migration->getSourcePlugin();
  393. $table = [];
  394. foreach ($source->fields() as $machine_name => $description) {
  395. $table[] = [strip_tags($description), $machine_name];
  396. }
  397. drush_print_table($table);
  398. }
  399. else {
  400. drush_log(dt('Migration @id does not exist', ['@id' => $migration_id]), 'error');
  401. }
  402. }
  403. /**
  404. * Retrieve a list of active migrations.
  405. *
  406. * @param string $migration_ids
  407. * Comma-separated list of migrations - if present, return only these migrations.
  408. *
  409. * @return MigrationInterface[][]
  410. * An array keyed by migration group, each value containing an array of
  411. * migrations or an empty array if no migrations match the input criteria.
  412. */
  413. function drush_migrate_tools_migration_list($migration_ids = '') {
  414. // Filter keys must match the migration configuration property name.
  415. $filter['migration_group'] = drush_get_option('group') ? explode(',', drush_get_option('group')) : [];
  416. $filter['migration_tags'] = drush_get_option('tag') ? explode(',', drush_get_option('tag')) : [];
  417. $manager = \Drupal::service('plugin.manager.config_entity_migration');
  418. $plugins = $manager->createInstances([]);
  419. $matched_migrations = [];
  420. // Get the set of migrations that may be filtered.
  421. if (empty($migration_ids)) {
  422. $matched_migrations = $plugins;
  423. }
  424. else {
  425. // Get the requested migrations.
  426. $migration_ids = explode(',', Unicode::strtolower($migration_ids));
  427. foreach ($plugins as $id => $migration) {
  428. if (in_array(Unicode::strtolower($id), $migration_ids)) {
  429. $matched_migrations [$id] = $migration;
  430. }
  431. }
  432. }
  433. // Filters the matched migrations if a group or a tag has been input.
  434. if (!empty($filter['migration_group']) || !empty($filter['migration_tags'])) {
  435. // Get migrations in any of the specified groups and with any of the
  436. // specified tags.
  437. foreach ($filter as $property => $values) {
  438. if (!empty($values)) {
  439. $filtered_migrations = [];
  440. foreach ($values as $search_value) {
  441. foreach ($matched_migrations as $id => $migration) {
  442. // Cast to array because migration_tags can be an array.
  443. $configured_values = (array) $migration->get($property);
  444. $configured_id = (in_array($search_value, $configured_values)) ? $search_value : 'default';
  445. if (empty($search_value) || $search_value == $configured_id) {
  446. if (empty($migration_ids) || in_array(Unicode::strtolower($id), $migration_ids)) {
  447. $filtered_migrations[$id] = $migration;
  448. }
  449. }
  450. }
  451. }
  452. $matched_migrations = $filtered_migrations;
  453. }
  454. }
  455. }
  456. // Sort the matched migrations by group.
  457. if (!empty($matched_migrations)) {
  458. foreach ($matched_migrations as $id => $migration) {
  459. $configured_group_id = empty($migration->get('migration_group')) ? 'default' : $migration->get('migration_group');
  460. $migrations[$configured_group_id][$id] = $migration;
  461. }
  462. }
  463. return isset($migrations) ? $migrations : [];
  464. }