/** * {@inheritdoc} */ public function saveIdMapping(Row $row, array $destination_id_values, $source_row_status = MigrateIdMapInterface::STATUS_IMPORTED, $rollback_action = MigrateIdMapInterface::ROLLBACK_DELETE) { // Construct the source key. $source_id_values = $row->getSourceIdValues(); // Construct the source key and initialize to empty variable keys. $keys = array(); foreach ($this->sourceIdFields() as $field_name => $key_name) { // A NULL key value will fail. if (!isset($source_id_values[$field_name])) { $this->message->display(t('Could not save to map table due to NULL value for key field @field', array('@field' => $field_name)), 'error'); return; } $keys[$key_name] = $source_id_values[$field_name]; } $fields = array('source_row_status' => (int) $source_row_status, 'rollback_action' => (int) $rollback_action, 'hash' => $row->getHash()); $count = 0; foreach ($destination_id_values as $dest_id) { $fields['destid' . ++$count] = $dest_id; } if ($count && $count != count($this->destinationIdFields())) { $this->message->display(t('Could not save to map table due to missing destination id values'), 'error'); return; } if ($this->migration->get('trackLastImported')) { $fields['last_imported'] = time(); } if ($keys) { // Notify anyone listening of the map row we're about to save. $this->eventDispatcher->dispatch(MigrateEvents::MAP_SAVE, new MigrateMapSaveEvent($this, $keys + $fields)); $this->getDatabase()->merge($this->mapTableName())->key($keys)->fields($fields)->execute(); } }
/** * {@inheritdoc} * * The migration iterates over rows returned by the source plugin. This * method determines the next row which will be processed and imported into * the system. * * The method tracks the source and destination IDs using the ID map plugin. * * This also takes care about highwater support. Highwater allows to reimport * rows from a previous migration run, which got changed in the meantime. * This is done by specifying a highwater field, which is compared with the * last time, the migration got executed (originalHighWater). */ public function next() { $this->currentSourceIds = NULL; $this->currentRow = NULL; // In order to find the next row we want to process, we ask the source // plugin for the next possible row. while (!isset($this->currentRow) && $this->getIterator()->valid()) { $row_data = $this->getIterator()->current() + $this->configuration; $this->getIterator()->next(); $row = new Row($row_data, $this->migration->getSourcePlugin()->getIds(), $this->migration->get('destinationIds')); // Populate the source key for this row. $this->currentSourceIds = $row->getSourceIdValues(); // Pick up the existing map row, if any, unless getNextRow() did it. if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentSourceIds))) { $row->setIdMap($id_map); } // Clear any previous messages for this row before potentially adding // new ones. if (!empty($this->currentSourceIds)) { $this->idMap->delete($this->currentSourceIds, TRUE); } // Preparing the row gives source plugins the chance to skip. if ($this->prepareRow($row) === FALSE) { continue; } // Check whether the row needs processing. // 1. This row has not been imported yet. // 2. Explicitly set to update. // 3. The row is newer than the current highwater mark. // 4. If no such property exists then try by checking the hash of the row. if (!$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) { $this->currentRow = $row->freezeSource(); } } }
/** * {@inheritdoc} * * The migration iterates over rows returned by the source plugin. This * method determines the next row which will be processed and imported into * the system. * * The method tracks the source and destination IDs using the ID map plugin. * * This also takes care about highwater support. Highwater allows to reimport * rows from a previous migration run, which got changed in the meantime. * This is done by specifying a highwater field, which is compared with the * last time, the migration got executed (originalHighWater). */ public function next() { $this->currentSourceIds = NULL; $this->currentRow = NULL; $source_configuration = $this->migration->get('source'); // In order to find the next row we want to process, we ask the source // plugin for the next possible row. while (!isset($this->currentRow) && $this->getIterator()->valid()) { $row_data = $this->getIterator()->current() + $source_configuration; $this->getIterator()->next(); $row = new Row($row_data, $this->migration->getSourcePlugin()->getIds(), $this->migration->get('destinationIds')); // Populate the source key for this row. $this->currentSourceIds = $row->getSourceIdValues(); // Pick up the existing map row, if any, unless getNextRow() did it. if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentSourceIds))) { $row->setIdMap($id_map); } // In case we have specified an ID list, but the ID given by the source is // not in there, we skip the row. $id_in_the_list = $this->idList && in_array(reset($this->currentSourceIds), $this->idList); if ($this->idList && !$id_in_the_list) { continue; } // Preparing the row gives source plugins the chance to skip. if ($this->prepareRow($row) === FALSE) { continue; } // Check whether the row needs processing. // 1. Explicitly specified IDs. // 2. This row has not been imported yet. // 3. Explicitly set to update. // 4. The row is newer than the current highwater mark. // 5. If no such property exists then try by checking the hash of the row. if ($id_in_the_list || !$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) { $this->currentRow = $row->freezeSource(); } } }
/** * Implementation of MigrateSource::performRewind(). * * We could simply execute the query and be functionally correct, but * we will take advantage of the PDO-based API to optimize the query up-front. */ protected function initializeIterator() { $this->prepareQuery(); $high_water_property = $this->migration->get('highWaterProperty'); // Get the key values, for potential use in joining to the map table, or // enforcing idlist. $keys = array(); // The rules for determining what conditions to add to the query are as // follows (applying first applicable rule) // 1. If idlist is provided, then only process items in that list (AND key // IN (idlist)). Only applicable with single-value keys. if ($id_list = $this->migration->get('idlist')) { $this->query->condition($keys[0], $id_list, 'IN'); } else { // 2. If the map is joinable, join it. We will want to accept all rows // which are either not in the map, or marked in the map as NEEDS_UPDATE. // Note that if high water fields are in play, we want to accept all rows // above the high water mark in addition to those selected by the map // conditions, so we need to OR them together (but AND with any existing // conditions in the query). So, ultimately the SQL condition will look // like (original conditions) AND (map IS NULL OR map needs update // OR above high water). $conditions = $this->query->orConditionGroup(); $condition_added = FALSE; if ($this->mapJoinable()) { // Build the join to the map table. Because the source key could have // multiple fields, we need to build things up. $count = 1; $map_join = ''; $delimiter = ''; foreach ($this->getIds() as $field_name => $field_schema) { if (isset($field_schema['alias'])) { $field_name = $field_schema['alias'] . '.' . $field_name; } $map_join .= "{$delimiter}{$field_name} = map.sourceid" . $count++; $delimiter = ' AND '; } $alias = $this->query->leftJoin($this->migration->getIdMap()->getQualifiedMapTableName(), 'map', $map_join); $conditions->isNull($alias . '.sourceid1'); $conditions->condition($alias . '.source_row_status', MigrateIdMapInterface::STATUS_NEEDS_UPDATE); $condition_added = TRUE; // And as long as we have the map table, add its data to the row. $n = count($this->getIds()); for ($count = 1; $count <= $n; $count++) { $map_key = 'sourceid' . $count; $this->query->addField($alias, $map_key, "migrate_map_{$map_key}"); } if ($n = count($this->migration->get('destinationIds'))) { for ($count = 1; $count <= $n; $count++) { $map_key = 'destid' . $count++; $this->query->addField($alias, $map_key, "migrate_map_{$map_key}"); } } $this->query->addField($alias, 'source_row_status', 'migrate_map_source_row_status'); } // 3. If we are using high water marks, also include rows above the mark. // But, include all rows if the high water mark is not set. if (isset($high_water_property['name']) && ($high_water = $this->migration->getHighWater()) !== '') { if (isset($high_water_property['alias'])) { $high_water = $high_water_property['alias'] . '.' . $high_water_property['name']; } else { $high_water = $high_water_property['name']; } $conditions->condition($high_water, $high_water, '>'); $condition_added = TRUE; } if ($condition_added) { $this->query->condition($conditions); } } return new \IteratorIterator($this->query->execute()); }
/** * Performs an import operation - migrate items from source to destination. */ public function import() { // Knock off migration if the requirements haven't been met. if (!$this->migration->checkRequirements()) { $this->message->display($this->t('Migration @id did not meet the requirements', array('@id' => $this->migration->id())), 'error'); return MigrationInterface::RESULT_FAILED; } $return = MigrationInterface::RESULT_COMPLETED; $source = $this->getSource(); $id_map = $this->migration->getIdMap(); try { $source->rewind(); } catch (\Exception $e) { $this->message->display($this->t('Migration failed with source plugin exception: !e', array('!e' => $e->getMessage())), 'error'); return MigrationInterface::RESULT_FAILED; } $destination = $this->migration->getDestinationPlugin(); while ($source->valid()) { $row = $source->current(); if ($this->sourceIdValues = $row->getSourceIdValues()) { // Wipe old messages, and save any new messages. $id_map->delete($this->sourceIdValues, TRUE); $this->saveQueuedMessages(); } try { $this->processRow($row); $save = TRUE; } catch (MigrateSkipRowException $e) { $id_map->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_IGNORED, $this->rollbackAction); $save = FALSE; } if ($save) { try { $destination_id_values = $destination->import($row, $id_map->lookupDestinationId($this->sourceIdValues)); if ($destination_id_values) { // We do not save an idMap entry for config. if ($destination_id_values !== TRUE) { $id_map->saveIdMapping($row, $destination_id_values, $this->sourceRowStatus, $this->rollbackAction); } $this->successesSinceFeedback++; $this->totalSuccesses++; } else { $id_map->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_FAILED, $this->rollbackAction); if (!$id_map->messageCount()) { $message = $this->t('New object was not saved, no error provided'); $this->saveMessage($message); $this->message->display($message); } } } catch (MigrateException $e) { $this->migration->getIdMap()->saveIdMapping($row, array(), $e->getStatus(), $this->rollbackAction); $this->saveMessage($e->getMessage(), $e->getLevel()); $this->message->display($e->getMessage(), 'error'); } catch (\Exception $e) { $this->migration->getIdMap()->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_FAILED, $this->rollbackAction); $this->handleException($e); } } $this->totalProcessed++; $this->processedSinceFeedback++; if ($highwater_property = $this->migration->get('highwaterProperty')) { $this->migration->saveHighwater($row->getSourceProperty($highwater_property['name'])); } // Reset row properties. unset($sourceValues, $destinationValues); $this->sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED; if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) { break; } if ($this->timeOptionExceeded()) { break; } try { $source->next(); } catch (\Exception $e) { $this->message->display($this->t('Migration failed with source plugin exception: !e', array('!e' => $e->getMessage())), 'error'); return MigrationInterface::RESULT_FAILED; } } /** * @TODO uncomment this */ #$this->progressMessage($return); return $return; }
/** * Builds a row for an entity in the entity listing. * * @param EntityInterface $migration * The entity for which to build the row. * * @return array * A render array of the table row for displaying the entity. * * @see Drupal\Core\Entity\EntityListController::render() */ public function buildRow(MigrationInterface $migration) { $row['label'] = $migration->label(); $row['machine_name'] = $migration->id(); $row['status'] = $migration->getStatusLabel(); // Derive the stats $source_plugin = $migration->getSourcePlugin(); $row['total'] = $source_plugin->count(); $map = $migration->getIdMap(); $row['imported'] = $map->importedCount(); // -1 indicates uncountable sources. if ($row['total'] == -1) { $row['total'] = $this->t('N/A'); $row['unprocessed'] = $this->t('N/A'); } else { $row['unprocessed'] = $row['total'] - $map->processedCount(); } $group = $migration->get('migration_group'); if (!$group) { $group = 'default'; } // @todo: This is most likely not a Best Practice (tm). $row['messages']['data']['#markup'] = '<a href="/admin/structure/migrate/manage/' . $group . '/migrations/' . $migration->id() . '/messages">' . $map->messageCount() . '</a>'; $migrate_last_imported_store = \Drupal::keyValue('migrate_last_imported'); $last_imported = $migrate_last_imported_store->get($migration->id(), FALSE); if ($last_imported) { /** @var DateFormatter $date_formatter */ $date_formatter = \Drupal::service('date.formatter'); $row['last_imported'] = $date_formatter->format($last_imported / 1000, 'custom', 'Y-m-d H:i:s'); } else { $row['last_imported'] = ''; } return $row; // + parent::buildRow($migration); }
/** * Class constructor. * * @param \Drupal\migrate\Entity\MigrationInterface $migration * The migration entity. * @param \Drupal\migrate\MigrateExecutable $migrate_executable * The migration executable. */ public function __construct(MigrationInterface $migration, MigrateExecutable $migrate_executable) { $this->migration = $migration; $this->migrateExecutable = $migrate_executable; $configuration = $migration->get('source'); if (!empty($configuration['cache_counts'])) { $this->cacheCounts = TRUE; } if (!empty($configuration['skip_count'])) { $this->skipCount = TRUE; } if (!empty($configuration['cache_key'])) { $this->cacheKey = $configuration['cache_key']; } if (!empty($configuration['track_changes'])) { $this->trackChanges = $configuration['track_changes']; } }