/**
* {@inheritdoc}
*/
public function saveIdMapping(Row $row, array $destination_id_values, $source_row_status = MigrateIdMapInterface::STATUS_IMPORTED, $rollback_action = MigrateIdMapInterface::ROLLBACK_DELETE)
{
// Construct the source key.
$source_id_values = $row->getSourceIdValues();
// Construct the source key and initialize to empty variable keys.
$keys = array();
foreach ($this->sourceIdFields() as $field_name => $key_name) {
// A NULL key value will fail.
if (!isset($source_id_values[$field_name])) {
$this->message->display(t('Could not save to map table due to NULL value for key field @field', array('@field' => $field_name)), 'error');
return;
}
$keys[$key_name] = $source_id_values[$field_name];
}
$fields = array('source_row_status' => (int) $source_row_status, 'rollback_action' => (int) $rollback_action, 'hash' => $row->getHash());
$count = 0;
foreach ($destination_id_values as $dest_id) {
$fields['destid' . ++$count] = $dest_id;
}
if ($count && $count != count($this->destinationIdFields())) {
$this->message->display(t('Could not save to map table due to missing destination id values'), 'error');
return;
}
if ($this->migration->get('trackLastImported')) {
$fields['last_imported'] = time();
}
if ($keys) {
// Notify anyone listening of the map row we're about to save.
$this->eventDispatcher->dispatch(MigrateEvents::MAP_SAVE, new MigrateMapSaveEvent($this, $keys + $fields));
$this->getDatabase()->merge($this->mapTableName())->key($keys)->fields($fields)->execute();
}
}
/**
* {@inheritdoc}
*
* The migration iterates over rows returned by the source plugin. This
* method determines the next row which will be processed and imported into
* the system.
*
* The method tracks the source and destination IDs using the ID map plugin.
*
* This also takes care about highwater support. Highwater allows to reimport
* rows from a previous migration run, which got changed in the meantime.
* This is done by specifying a highwater field, which is compared with the
* last time, the migration got executed (originalHighWater).
*/
public function next()
{
$this->currentSourceIds = NULL;
$this->currentRow = NULL;
// In order to find the next row we want to process, we ask the source
// plugin for the next possible row.
while (!isset($this->currentRow) && $this->getIterator()->valid()) {
$row_data = $this->getIterator()->current() + $this->configuration;
$this->getIterator()->next();
$row = new Row($row_data, $this->migration->getSourcePlugin()->getIds(), $this->migration->get('destinationIds'));
// Populate the source key for this row.
$this->currentSourceIds = $row->getSourceIdValues();
// Pick up the existing map row, if any, unless getNextRow() did it.
if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentSourceIds))) {
$row->setIdMap($id_map);
}
// Clear any previous messages for this row before potentially adding
// new ones.
if (!empty($this->currentSourceIds)) {
$this->idMap->delete($this->currentSourceIds, TRUE);
}
// Preparing the row gives source plugins the chance to skip.
if ($this->prepareRow($row) === FALSE) {
continue;
}
// Check whether the row needs processing.
// 1. This row has not been imported yet.
// 2. Explicitly set to update.
// 3. The row is newer than the current highwater mark.
// 4. If no such property exists then try by checking the hash of the row.
if (!$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) {
$this->currentRow = $row->freezeSource();
}
}
}
/**
* {@inheritdoc}
*
* The migration iterates over rows returned by the source plugin. This
* method determines the next row which will be processed and imported into
* the system.
*
* The method tracks the source and destination IDs using the ID map plugin.
*
* This also takes care about highwater support. Highwater allows to reimport
* rows from a previous migration run, which got changed in the meantime.
* This is done by specifying a highwater field, which is compared with the
* last time, the migration got executed (originalHighWater).
*/
public function next()
{
$this->currentSourceIds = NULL;
$this->currentRow = NULL;
$source_configuration = $this->migration->get('source');
// In order to find the next row we want to process, we ask the source
// plugin for the next possible row.
while (!isset($this->currentRow) && $this->getIterator()->valid()) {
$row_data = $this->getIterator()->current() + $source_configuration;
$this->getIterator()->next();
$row = new Row($row_data, $this->migration->getSourcePlugin()->getIds(), $this->migration->get('destinationIds'));
// Populate the source key for this row.
$this->currentSourceIds = $row->getSourceIdValues();
// Pick up the existing map row, if any, unless getNextRow() did it.
if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentSourceIds))) {
$row->setIdMap($id_map);
}
// In case we have specified an ID list, but the ID given by the source is
// not in there, we skip the row.
$id_in_the_list = $this->idList && in_array(reset($this->currentSourceIds), $this->idList);
if ($this->idList && !$id_in_the_list) {
continue;
}
// Preparing the row gives source plugins the chance to skip.
if ($this->prepareRow($row) === FALSE) {
continue;
}
// Check whether the row needs processing.
// 1. Explicitly specified IDs.
// 2. This row has not been imported yet.
// 3. Explicitly set to update.
// 4. The row is newer than the current highwater mark.
// 5. If no such property exists then try by checking the hash of the row.
if ($id_in_the_list || !$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) {
$this->currentRow = $row->freezeSource();
}
}
}
/**
* Implementation of MigrateSource::performRewind().
*
* We could simply execute the query and be functionally correct, but
* we will take advantage of the PDO-based API to optimize the query up-front.
*/
protected function initializeIterator()
{
$this->prepareQuery();
$high_water_property = $this->migration->get('highWaterProperty');
// Get the key values, for potential use in joining to the map table, or
// enforcing idlist.
$keys = array();
// The rules for determining what conditions to add to the query are as
// follows (applying first applicable rule)
// 1. If idlist is provided, then only process items in that list (AND key
// IN (idlist)). Only applicable with single-value keys.
if ($id_list = $this->migration->get('idlist')) {
$this->query->condition($keys[0], $id_list, 'IN');
} else {
// 2. If the map is joinable, join it. We will want to accept all rows
// which are either not in the map, or marked in the map as NEEDS_UPDATE.
// Note that if high water fields are in play, we want to accept all rows
// above the high water mark in addition to those selected by the map
// conditions, so we need to OR them together (but AND with any existing
// conditions in the query). So, ultimately the SQL condition will look
// like (original conditions) AND (map IS NULL OR map needs update
// OR above high water).
$conditions = $this->query->orConditionGroup();
$condition_added = FALSE;
if ($this->mapJoinable()) {
// Build the join to the map table. Because the source key could have
// multiple fields, we need to build things up.
$count = 1;
$map_join = '';
$delimiter = '';
foreach ($this->getIds() as $field_name => $field_schema) {
if (isset($field_schema['alias'])) {
$field_name = $field_schema['alias'] . '.' . $field_name;
}
$map_join .= "{$delimiter}{$field_name} = map.sourceid" . $count++;
$delimiter = ' AND ';
}
$alias = $this->query->leftJoin($this->migration->getIdMap()->getQualifiedMapTableName(), 'map', $map_join);
$conditions->isNull($alias . '.sourceid1');
$conditions->condition($alias . '.source_row_status', MigrateIdMapInterface::STATUS_NEEDS_UPDATE);
$condition_added = TRUE;
// And as long as we have the map table, add its data to the row.
$n = count($this->getIds());
for ($count = 1; $count <= $n; $count++) {
$map_key = 'sourceid' . $count;
$this->query->addField($alias, $map_key, "migrate_map_{$map_key}");
}
if ($n = count($this->migration->get('destinationIds'))) {
for ($count = 1; $count <= $n; $count++) {
$map_key = 'destid' . $count++;
$this->query->addField($alias, $map_key, "migrate_map_{$map_key}");
}
}
$this->query->addField($alias, 'source_row_status', 'migrate_map_source_row_status');
}
// 3. If we are using high water marks, also include rows above the mark.
// But, include all rows if the high water mark is not set.
if (isset($high_water_property['name']) && ($high_water = $this->migration->getHighWater()) !== '') {
if (isset($high_water_property['alias'])) {
$high_water = $high_water_property['alias'] . '.' . $high_water_property['name'];
} else {
$high_water = $high_water_property['name'];
}
$conditions->condition($high_water, $high_water, '>');
$condition_added = TRUE;
}
if ($condition_added) {
$this->query->condition($conditions);
}
}
return new \IteratorIterator($this->query->execute());
}
/**
* Performs an import operation - migrate items from source to destination.
*/
public function import()
{
// Knock off migration if the requirements haven't been met.
if (!$this->migration->checkRequirements()) {
$this->message->display($this->t('Migration @id did not meet the requirements', array('@id' => $this->migration->id())), 'error');
return MigrationInterface::RESULT_FAILED;
}
$return = MigrationInterface::RESULT_COMPLETED;
$source = $this->getSource();
$id_map = $this->migration->getIdMap();
try {
$source->rewind();
} catch (\Exception $e) {
$this->message->display($this->t('Migration failed with source plugin exception: !e', array('!e' => $e->getMessage())), 'error');
return MigrationInterface::RESULT_FAILED;
}
$destination = $this->migration->getDestinationPlugin();
while ($source->valid()) {
$row = $source->current();
if ($this->sourceIdValues = $row->getSourceIdValues()) {
// Wipe old messages, and save any new messages.
$id_map->delete($this->sourceIdValues, TRUE);
$this->saveQueuedMessages();
}
try {
$this->processRow($row);
$save = TRUE;
} catch (MigrateSkipRowException $e) {
$id_map->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_IGNORED, $this->rollbackAction);
$save = FALSE;
}
if ($save) {
try {
$destination_id_values = $destination->import($row, $id_map->lookupDestinationId($this->sourceIdValues));
if ($destination_id_values) {
// We do not save an idMap entry for config.
if ($destination_id_values !== TRUE) {
$id_map->saveIdMapping($row, $destination_id_values, $this->sourceRowStatus, $this->rollbackAction);
}
$this->successesSinceFeedback++;
$this->totalSuccesses++;
} else {
$id_map->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_FAILED, $this->rollbackAction);
if (!$id_map->messageCount()) {
$message = $this->t('New object was not saved, no error provided');
$this->saveMessage($message);
$this->message->display($message);
}
}
} catch (MigrateException $e) {
$this->migration->getIdMap()->saveIdMapping($row, array(), $e->getStatus(), $this->rollbackAction);
$this->saveMessage($e->getMessage(), $e->getLevel());
$this->message->display($e->getMessage(), 'error');
} catch (\Exception $e) {
$this->migration->getIdMap()->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_FAILED, $this->rollbackAction);
$this->handleException($e);
}
}
$this->totalProcessed++;
$this->processedSinceFeedback++;
if ($highwater_property = $this->migration->get('highwaterProperty')) {
$this->migration->saveHighwater($row->getSourceProperty($highwater_property['name']));
}
// Reset row properties.
unset($sourceValues, $destinationValues);
$this->sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED;
if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) {
break;
}
if ($this->timeOptionExceeded()) {
break;
}
try {
$source->next();
} catch (\Exception $e) {
$this->message->display($this->t('Migration failed with source plugin exception: !e', array('!e' => $e->getMessage())), 'error');
return MigrationInterface::RESULT_FAILED;
}
}
/**
* @TODO uncomment this
*/
#$this->progressMessage($return);
return $return;
}
/**
* Builds a row for an entity in the entity listing.
*
* @param EntityInterface $migration
* The entity for which to build the row.
*
* @return array
* A render array of the table row for displaying the entity.
*
* @see Drupal\Core\Entity\EntityListController::render()
*/
public function buildRow(MigrationInterface $migration)
{
$row['label'] = $migration->label();
$row['machine_name'] = $migration->id();
$row['status'] = $migration->getStatusLabel();
// Derive the stats
$source_plugin = $migration->getSourcePlugin();
$row['total'] = $source_plugin->count();
$map = $migration->getIdMap();
$row['imported'] = $map->importedCount();
// -1 indicates uncountable sources.
if ($row['total'] == -1) {
$row['total'] = $this->t('N/A');
$row['unprocessed'] = $this->t('N/A');
} else {
$row['unprocessed'] = $row['total'] - $map->processedCount();
}
$group = $migration->get('migration_group');
if (!$group) {
$group = 'default';
}
// @todo: This is most likely not a Best Practice (tm).
$row['messages']['data']['#markup'] = '<a href="/admin/structure/migrate/manage/' . $group . '/migrations/' . $migration->id() . '/messages">' . $map->messageCount() . '</a>';
$migrate_last_imported_store = \Drupal::keyValue('migrate_last_imported');
$last_imported = $migrate_last_imported_store->get($migration->id(), FALSE);
if ($last_imported) {
/** @var DateFormatter $date_formatter */
$date_formatter = \Drupal::service('date.formatter');
$row['last_imported'] = $date_formatter->format($last_imported / 1000, 'custom', 'Y-m-d H:i:s');
} else {
$row['last_imported'] = '';
}
return $row;
// + parent::buildRow($migration);
}
/**
* Class constructor.
*
* @param \Drupal\migrate\Entity\MigrationInterface $migration
* The migration entity.
* @param \Drupal\migrate\MigrateExecutable $migrate_executable
* The migration executable.
*/
public function __construct(MigrationInterface $migration, MigrateExecutable $migrate_executable)
{
$this->migration = $migration;
$this->migrateExecutable = $migrate_executable;
$configuration = $migration->get('source');
if (!empty($configuration['cache_counts'])) {
$this->cacheCounts = TRUE;
}
if (!empty($configuration['skip_count'])) {
$this->skipCount = TRUE;
}
if (!empty($configuration['cache_key'])) {
$this->cacheKey = $configuration['cache_key'];
}
if (!empty($configuration['track_changes'])) {
$this->trackChanges = $configuration['track_changes'];
}
}
