/**
* {@inheritdoc}
*
* The migration iterates over rows returned by the source plugin. This
* method determines the next row which will be processed and imported into
* the system.
*
* The method tracks the source and destination IDs using the ID map plugin.
*
* This also takes care about highwater support. Highwater allows to reimport
* rows from a previous migration run, which got changed in the meantime.
* This is done by specifying a highwater field, which is compared with the
* last time, the migration got executed (originalHighWater).
*/
public function next()
{
$this->currentSourceIds = NULL;
$this->currentRow = NULL;
// In order to find the next row we want to process, we ask the source
// plugin for the next possible row.
while (!isset($this->currentRow) && $this->getIterator()->valid()) {
$row_data = $this->getIterator()->current() + $this->configuration;
$this->getIterator()->next();
$row = new Row($row_data, $this->migration->getSourcePlugin()->getIds(), $this->migration->get('destinationIds'));
// Populate the source key for this row.
$this->currentSourceIds = $row->getSourceIdValues();
// Pick up the existing map row, if any, unless getNextRow() did it.
if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentSourceIds))) {
$row->setIdMap($id_map);
}
// Clear any previous messages for this row before potentially adding
// new ones.
if (!empty($this->currentSourceIds)) {
$this->idMap->delete($this->currentSourceIds, TRUE);
}
// Preparing the row gives source plugins the chance to skip.
if ($this->prepareRow($row) === FALSE) {
continue;
}
// Check whether the row needs processing.
// 1. This row has not been imported yet.
// 2. Explicitly set to update.
// 3. The row is newer than the current highwater mark.
// 4. If no such property exists then try by checking the hash of the row.
if (!$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) {
$this->currentRow = $row->freezeSource();
}
}
}
/**
* {@inheritdoc}
*/
public function deleteBulk(array $source_id_values)
{
// If we have a single-column key, we can shortcut it.
if (count($this->migration->getSourcePlugin()->getIds()) == 1) {
$sourceids = array();
foreach ($source_id_values as $source_id) {
// Notify anyone listening of the map rows we're about to delete.
$this->eventDispatcher->dispatch(MigrateEvents::MAP_DELETE, new MigrateMapDeleteEvent($this, $source_id));
$sourceids[] = $source_id;
}
$this->getDatabase()->delete($this->mapTableName())->condition('sourceid1', $sourceids, 'IN')->execute();
$this->getDatabase()->delete($this->messageTableName())->condition('sourceid1', $sourceids, 'IN')->execute();
} else {
foreach ($source_id_values as $source_id) {
// Notify anyone listening of the map rows we're deleting.
$this->eventDispatcher->dispatch(MigrateEvents::MAP_DELETE, new MigrateMapDeleteEvent($this, $source_id));
$map_query = $this->getDatabase()->delete($this->mapTableName());
$message_query = $this->getDatabase()->delete($this->messageTableName());
$count = 1;
foreach ($source_id as $key_value) {
$map_query->condition('sourceid' . $count, $key_value);
$message_query->condition('sourceid' . $count++, $key_value);
}
$map_query->execute();
$message_query->execute();
}
}
}
/**
* Create the map and message tables if they don't already exist.
*/
protected function ensureTables()
{
if (!$this->getDatabase()->schema()->tableExists($this->mapTableName)) {
// Generate appropriate schema info for the map and message tables,
// and map from the source field names to the map/msg field names.
$count = 1;
$source_id_schema = array();
foreach ($this->migration->getSourcePlugin()->getIds() as $id_definition) {
$mapkey = 'sourceid' . $count++;
$source_id_schema[$mapkey] = $this->getFieldSchema($id_definition);
$source_id_schema[$mapkey]['not null'] = TRUE;
}
$source_ids_hash[static::SOURCE_IDS_HASH] = array('type' => 'varchar', 'length' => '64', 'not null' => TRUE, 'description' => 'Hash of source ids. Used as primary key');
$fields = $source_ids_hash + $source_id_schema;
// Add destination identifiers to map table.
// @todo How do we discover the destination schema?
$count = 1;
foreach ($this->migration->getDestinationPlugin()->getIds() as $id_definition) {
// Allow dest identifier fields to be NULL (for IGNORED/FAILED cases).
$mapkey = 'destid' . $count++;
$fields[$mapkey] = $this->getFieldSchema($id_definition);
$fields[$mapkey]['not null'] = FALSE;
}
$fields['source_row_status'] = array('type' => 'int', 'size' => 'tiny', 'unsigned' => TRUE, 'not null' => TRUE, 'default' => MigrateIdMapInterface::STATUS_IMPORTED, 'description' => 'Indicates current status of the source row');
$fields['rollback_action'] = array('type' => 'int', 'size' => 'tiny', 'unsigned' => TRUE, 'not null' => TRUE, 'default' => MigrateIdMapInterface::ROLLBACK_DELETE, 'description' => 'Flag indicating what to do for this item on rollback');
$fields['last_imported'] = array('type' => 'int', 'unsigned' => TRUE, 'not null' => TRUE, 'default' => 0, 'description' => 'UNIX timestamp of the last time this row was imported');
$fields['hash'] = array('type' => 'varchar', 'length' => '64', 'not null' => FALSE, 'description' => 'Hash of source row data, for detecting changes');
$schema = array('description' => 'Mappings from source identifier value(s) to destination identifier value(s).', 'fields' => $fields, 'primary key' => array(static::SOURCE_IDS_HASH));
$this->getDatabase()->schema()->createTable($this->mapTableName, $schema);
// Now do the message table.
if (!$this->getDatabase()->schema()->tableExists($this->messageTableName())) {
$fields = array();
$fields['msgid'] = array('type' => 'serial', 'unsigned' => TRUE, 'not null' => TRUE);
$fields += $source_ids_hash;
$fields['level'] = array('type' => 'int', 'unsigned' => TRUE, 'not null' => TRUE, 'default' => 1);
$fields['message'] = array('type' => 'text', 'size' => 'medium', 'not null' => TRUE);
$schema = array('description' => 'Messages generated during a migration process', 'fields' => $fields, 'primary key' => array('msgid'));
$this->getDatabase()->schema()->createTable($this->messageTableName(), $schema);
}
} else {
// Add any missing columns to the map table.
if (!$this->getDatabase()->schema()->fieldExists($this->mapTableName, 'rollback_action')) {
$this->getDatabase()->schema()->addField($this->mapTableName, 'rollback_action', array('type' => 'int', 'size' => 'tiny', 'unsigned' => TRUE, 'not null' => TRUE, 'default' => 0, 'description' => 'Flag indicating what to do for this item on rollback'));
}
if (!$this->getDatabase()->schema()->fieldExists($this->mapTableName, 'hash')) {
$this->getDatabase()->schema()->addField($this->mapTableName, 'hash', array('type' => 'varchar', 'length' => '64', 'not null' => FALSE, 'description' => 'Hash of source row data, for detecting changes'));
}
if (!$this->getDatabase()->schema()->fieldExists($this->mapTableName, static::SOURCE_IDS_HASH)) {
$this->getDatabase()->schema()->addField($this->mapTableName, static::SOURCE_IDS_HASH, array('type' => 'varchar', 'length' => '64', 'not null' => TRUE, 'description' => 'Hash of source ids. Used as primary key'));
}
}
}
/**
* {@inheritdoc}
*
* The migration iterates over rows returned by the source plugin. This
* method determines the next row which will be processed and imported into
* the system.
*
* The method tracks the source and destination IDs using the ID map plugin.
*
* This also takes care about highwater support. Highwater allows to reimport
* rows from a previous migration run, which got changed in the meantime.
* This is done by specifying a highwater field, which is compared with the
* last time, the migration got executed (originalHighWater).
*/
public function next()
{
$this->currentSourceIds = NULL;
$this->currentRow = NULL;
$source_configuration = $this->migration->get('source');
// In order to find the next row we want to process, we ask the source
// plugin for the next possible row.
while (!isset($this->currentRow) && $this->getIterator()->valid()) {
$row_data = $this->getIterator()->current() + $source_configuration;
$this->getIterator()->next();
$row = new Row($row_data, $this->migration->getSourcePlugin()->getIds(), $this->migration->get('destinationIds'));
// Populate the source key for this row.
$this->currentSourceIds = $row->getSourceIdValues();
// Pick up the existing map row, if any, unless getNextRow() did it.
if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentSourceIds))) {
$row->setIdMap($id_map);
}
// In case we have specified an ID list, but the ID given by the source is
// not in there, we skip the row.
$id_in_the_list = $this->idList && in_array(reset($this->currentSourceIds), $this->idList);
if ($this->idList && !$id_in_the_list) {
continue;
}
// Preparing the row gives source plugins the chance to skip.
if ($this->prepareRow($row) === FALSE) {
continue;
}
// Check whether the row needs processing.
// 1. Explicitly specified IDs.
// 2. This row has not been imported yet.
// 3. Explicitly set to update.
// 4. The row is newer than the current highwater mark.
// 5. If no such property exists then try by checking the hash of the row.
if ($id_in_the_list || !$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) {
$this->currentRow = $row->freezeSource();
}
}
}
/**
* {@inheritdoc}
*/
public function deleteBulk(array $source_id_values)
{
// If we have a single-column key, we can shortcut it.
if (count($this->migration->getSourcePlugin()->getIds()) == 1) {
$sourceids = array();
foreach ($source_id_values as $source_id) {
$sourceids[] = $source_id;
}
$this->getDatabase()->delete($this->mapTableName())->condition('sourceid1', $sourceids, 'IN')->execute();
$this->getDatabase()->delete($this->messageTableName())->condition('sourceid1', $sourceids, 'IN')->execute();
} else {
foreach ($source_id_values as $source_id) {
$map_query = $this->getDatabase()->delete($this->mapTableName());
$message_query = $this->getDatabase()->delete($this->messageTableName());
$count = 1;
foreach ($source_id as $key_value) {
$map_query->condition('sourceid' . $count, $key_value);
$message_query->condition('sourceid' . $count++, $key_value);
}
$map_query->execute();
$message_query->execute();
}
}
}
/**
* Builds a row for an entity in the entity listing.
*
* @param EntityInterface $migration
* The entity for which to build the row.
*
* @return array
* A render array of the table row for displaying the entity.
*
* @see Drupal\Core\Entity\EntityListController::render()
*/
public function buildRow(MigrationInterface $migration)
{
$row['label'] = $migration->label();
$row['machine_name'] = $migration->id();
$row['status'] = $migration->getStatusLabel();
// Derive the stats
$source_plugin = $migration->getSourcePlugin();
$row['total'] = $source_plugin->count();
$map = $migration->getIdMap();
$row['imported'] = $map->importedCount();
// -1 indicates uncountable sources.
if ($row['total'] == -1) {
$row['total'] = $this->t('N/A');
$row['unprocessed'] = $this->t('N/A');
} else {
$row['unprocessed'] = $row['total'] - $map->processedCount();
}
$group = $migration->get('migration_group');
if (!$group) {
$group = 'default';
}
// @todo: This is most likely not a Best Practice (tm).
$row['messages']['data']['#markup'] = '<a href="/admin/structure/migrate/manage/' . $group . '/migrations/' . $migration->id() . '/messages">' . $map->messageCount() . '</a>';
$migrate_last_imported_store = \Drupal::keyValue('migrate_last_imported');
$last_imported = $migrate_last_imported_store->get($migration->id(), FALSE);
if ($last_imported) {
/** @var DateFormatter $date_formatter */
$date_formatter = \Drupal::service('date.formatter');
$row['last_imported'] = $date_formatter->format($last_imported / 1000, 'custom', 'Y-m-d H:i:s');
} else {
$row['last_imported'] = '';
}
return $row;
// + parent::buildRow($migration);
}
