/**
* {@inheritdoc}
*/
public function perform()
{
$destinationDocument = $this->destination->getDocument(self::CONFIG_TABLE_NAME_DESTINATION);
$recordsCountSource = $this->source->getRecordsCount(self::CONFIG_TABLE_NAME_SOURCE);
$recordsCountDestination = $this->destination->getRecordsCount(self::CONFIG_TABLE_NAME_DESTINATION);
$this->progress->start($recordsCountSource);
$sourceRecords = $this->source->getRecords(self::CONFIG_TABLE_NAME_SOURCE, 0, $recordsCountSource);
$destinationRecords = $this->destination->getRecords(self::CONFIG_TABLE_NAME_DESTINATION, 0, $recordsCountDestination);
foreach ($sourceRecords as $sourceRecord) {
$this->progress->advance();
if (!$this->readerSettings->isNodeIgnored($sourceRecord[self::CONFIG_FIELD_PATH])) {
$sourceRecordPathMapped = $this->readerSettings->getNodeMap($sourceRecord[self::CONFIG_FIELD_PATH]);
foreach ($destinationRecords as &$destinationRecord) {
if ($destinationRecord[self::CONFIG_FIELD_SCOPE] == $sourceRecord[self::CONFIG_FIELD_SCOPE] && $destinationRecord[self::CONFIG_FIELD_SCOPE_ID] == $sourceRecord[self::CONFIG_FIELD_SCOPE_ID] && $destinationRecord[self::CONFIG_FIELD_PATH] == $sourceRecordPathMapped) {
$record = $this->applyHandler($destinationDocument, $sourceRecord, $destinationRecord);
$destinationRecord[self::CONFIG_FIELD_VALUE] = $record->getValue(self::CONFIG_FIELD_VALUE);
continue 2;
}
}
$record = $this->applyHandler($destinationDocument, $sourceRecord, []);
$record->setValue(self::CONFIG_FIELD_PATH, $sourceRecordPathMapped);
$destinationRecords[] = $record->getData();
}
}
foreach ($destinationRecords as &$destinationRecord) {
unset($destinationRecord[self::CONFIG_FIELD_CONFIG_ID]);
}
$this->destination->clearDocument(self::CONFIG_TABLE_NAME_DESTINATION);
$this->destination->saveRecords(self::CONFIG_TABLE_NAME_DESTINATION, $destinationRecords);
$this->progress->finish();
return true;
}
/**
* @return bool
*/
public function perform()
{
// catalog_product_entity_tier_price should be migrated first to save same value_id as into magento1
$sourceDocuments = array_keys($this->helper->getSourceDocumentFields());
$this->progress->start(count($sourceDocuments), LogManager::LOG_LEVEL_INFO);
$destinationName = $this->helper->getDestinationName();
$this->destination->clearDocument($destinationName);
$destDocument = $this->destination->getDocument($destinationName);
foreach ($sourceDocuments as $sourceDocName) {
$pageNumber = 0;
$this->logger->debug('migrating', ['table' => $sourceDocName]);
$this->progress->start($this->source->getRecordsCount($sourceDocName), LogManager::LOG_LEVEL_DEBUG);
while (!empty($items = $this->source->getRecords($sourceDocName, $pageNumber))) {
$pageNumber++;
$destinationRecords = $destDocument->getRecords();
foreach ($items as $recordData) {
unset($recordData['value_id']);
$this->progress->advance(LogManager::LOG_LEVEL_INFO);
$this->progress->advance(LogManager::LOG_LEVEL_DEBUG);
/** @var Record $destRecord */
$destRecord = $this->recordFactory->create(['document' => $destDocument, 'data' => $recordData]);
$destinationRecords->addRecord($destRecord);
}
$this->destination->saveRecords($destinationName, $destinationRecords);
}
$this->progress->finish(LogManager::LOG_LEVEL_DEBUG);
}
$this->progress->finish(LogManager::LOG_LEVEL_INFO);
return true;
}
/**
* @return void
*/
public function testGetRecordsCount()
{
$sourceCount = $this->source->getRecordsCount('table_with_data');
$destinationCount = $this->destination->getRecordsCount('table_without_data');
$this->assertEquals(7, $sourceCount);
$this->assertEquals(0, $destinationCount);
}
/**
* {@inheritdoc}
*/
public function perform()
{
$this->progress->start(count($this->helper->getDocumentList()));
foreach ($this->helper->getDocumentList() as $sourceName => $destinationName) {
$this->progress->advance();
if ($this->source->getRecordsCount($sourceName) != $this->destination->getRecordsCount($destinationName)) {
$this->errors[] = 'Mismatch of entities in the document: ' . $destinationName;
}
}
$this->progress->finish();
return $this->checkForErrors(Logger::ERROR);
}
/**
* @return bool
*/
public function perform()
{
$this->progressBar->start($this->getIterationsCount());
foreach ($this->helper->getDocumentList() as $sourceDocumentName => $destinationDocumentName) {
$this->progressBar->advance();
$sourceRecordsCount = $this->source->getRecordsCount($sourceDocumentName);
$destinationRecordsCount = $this->destination->getRecordsCount($destinationDocumentName);
if ($sourceRecordsCount != $destinationRecordsCount) {
$this->errors[] = 'Mismatch of entities in the document: ' . $destinationDocumentName;
}
}
$this->progressBar->finish();
return $this->checkForErrors();
}
/**
* {@inheritdoc}
*/
public function perform()
{
$this->progress->start(count($this->helper->getSourceDocumentFields()));
$sourceTotal = 0;
$destinationTotal = $this->destination->getRecordsCount($this->helper->getDestinationName());
foreach (array_keys($this->helper->getSourceDocumentFields()) as $sourceName) {
$sourceTotal += $this->source->getRecordsCount($sourceName);
$this->progress->advance();
}
if ($sourceTotal != $destinationTotal) {
$this->errors[] = 'Mismatch of amount of entities in documents';
}
$this->progress->finish();
return $this->checkForErrors(Logger::ERROR);
}
/**
* @return int
*/
protected function getIterationsCount()
{
$iterations = 0;
$document = $this->helper->getDocumentList();
$iterations += $this->source->getRecordsCount($document['source']);
return $iterations;
}
/**
* Get iterations count for step
*
* @return int
*/
protected function getIterationsCount()
{
$iterations = 0;
foreach (array_keys($this->readerGroups->getGroup('source_documents')) as $document) {
$iterations += $this->source->getRecordsCount($document);
}
return $iterations;
}
/**
* Get iterations count for step
*
* @return int
*/
protected function getIterationsCount()
{
$iterations = 0;
foreach (array_keys($this->helper->getDocumentList()) as $document) {
$iterations += $this->source->getRecordsCount($document);
}
return $iterations;
}
/**
* @param string $sourceDocName
* @param string $destinationName
* @return void
*/
protected function checkMapEntities($sourceDocName, $destinationName)
{
$sourceCount = $this->source->getRecordsCount($sourceDocName);
$destinationCount = $this->destination->getRecordsCount($destinationName);
if ($sourceCount != $destinationCount) {
$this->errors[] = sprintf('Mismatch of entities in the document: %s', $destinationName);
}
}
/**
* Volume check
*
* @return bool
*/
public function perform()
{
$sourceDocuments = array_keys($this->groups->getGroup('source_documents'));
$this->progress->start(count($sourceDocuments));
foreach ($sourceDocuments as $sourceName) {
$this->progress->advance();
$destinationName = $this->map->getDocumentMap($sourceName, MapInterface::TYPE_SOURCE);
$sourceFields = $this->source->getDocument($sourceName)->getStructure()->getFields();
$destinationFields = $this->destination->getDocument($destinationName)->getStructure()->getFields();
if (!empty(array_diff_key($sourceFields, $destinationFields))) {
$this->errors[] = 'Mismatch of fields in the document: ' . $destinationName;
}
if ($this->source->getRecordsCount($sourceName) != $this->destination->getRecordsCount($destinationName)) {
$this->errors[] = 'Mismatch of entities in the document: ' . $destinationName;
}
}
$this->progress->finish();
return $this->checkForErrors();
}
/**
* @return bool
*/
public function perform()
{
$sourceDocuments = $this->source->getDocumentList();
$this->progressBar->start(count($sourceDocuments));
foreach ($sourceDocuments as $sourceDocName) {
$this->progressBar->advance();
$destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE);
if (!$destinationName) {
continue;
}
$sourceCount = $this->source->getRecordsCount($sourceDocName);
$destinationCount = $this->destination->getRecordsCount($destinationName);
if ($sourceCount != $destinationCount) {
$this->errors[] = 'Mismatch of entities in the document: ' . $destinationName;
}
}
$this->progressBar->finish();
return $this->checkForErrors();
}
/**
* Volume check
*
* @return bool
*/
protected function volume()
{
$result = true;
$this->progress->start(1);
$result &= $this->source->getRecordsCount(self::SOURCE) + $this->countCmsPageRewrites(true) == $this->destination->getRecordsCount(self::DESTINATION);
if (!$result) {
$this->logger->error('Mismatch of entities in the document: url_rewrite');
}
$this->progress->advance();
$this->progress->finish();
return (bool) $result;
}
/**
* @param string $document
* @return int
*/
public function getSourceRecordsCount($document)
{
if ($this->sourceDocuments[$document] == 'entity_id') {
return $this->source->getRecordsCount($document);
}
$attributeType = $this->getAttributeType($document);
/** @var Mysql $adapter */
$adapter = $this->source->getAdapter();
$query = $adapter->getSelect()->from(['et' => $this->source->addDocumentPrefix($document)], 'COUNT(*)')->where('et.attribute_id NOT IN (?)', array_keys($this->skipAttributes[$attributeType]));
$count = $query->getAdapter()->fetchOne($query);
return $count;
}
/**
* @return bool
*/
public function perform()
{
$sourceDocuments = array_keys($this->readerGroups->getGroup('source_documents'));
$this->progress->start($this->getIterationsCount());
foreach ($sourceDocuments as $sourceDocName) {
$this->progress->advance();
$destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE);
if (!$destinationName) {
continue;
}
$sourceCount = $this->source->getRecordsCount($sourceDocName);
$destinationCount = $this->destination->getRecordsCount($destinationName);
if ($sourceCount != $destinationCount) {
$this->errors[] = 'Mismatch of entities in the document: ' . $destinationName;
}
}
if (!$this->checkCleared(array_keys($this->readerGroups->getGroup('destination_documents_to_clear')))) {
$this->errors[] = 'Log documents in the destination resource are not cleared';
}
$this->progress->finish();
return $this->checkForErrors();
}
/**
* {@inheritdoc}
*/
public function perform()
{
$updateData = $this->helper->getUpdateData();
$selectData = $this->helper->getSelectData();
$sourceDocuments = $this->readerGroups->getGroup('source_documents');
foreach ($sourceDocuments as $sourceDocName => $idKey) {
if ($this->source->getRecordsCount($this->source->getDeltaLogName($sourceDocName)) == 0) {
continue;
}
$items = $this->source->getChangedRecords($sourceDocName, $idKey, 0, true);
if (empty($items)) {
continue;
}
$this->logger->debug(sprintf('%s has changes', $sourceDocName));
if (!$this->eolOnce) {
$this->eolOnce = true;
echo PHP_EOL;
}
$gridIdKey = $updateData[$sourceDocName]['idKey'];
$page = 1;
do {
$ids = [];
foreach ($items as $data) {
echo '.';
$ids[] = $data[$gridIdKey];
}
foreach ($updateData[$sourceDocName]['methods'] as $method) {
echo '.';
$destinationDocumentName = $selectData[$method]['destination'];
$select = call_user_func_array([$this->data, $method], [$selectData[$method]['columns'], $ids]);
$this->destination->getAdapter()->insertFromSelect($select, $this->destination->addDocumentPrefix($destinationDocumentName), [], \Magento\Framework\Db\Adapter\AdapterInterface::INSERT_ON_DUPLICATE);
}
$documentNameDelta = $this->source->getDeltaLogName($sourceDocName);
$documentNameDelta = $this->source->addDocumentPrefix($documentNameDelta);
$this->markRecordsProcessed($documentNameDelta, $idKey, $ids);
} while (!empty($items = $this->source->getChangedRecords($sourceDocName, $idKey, $page++)));
}
return true;
}
/**
* @return bool
*/
public function perform()
{
$this->progressBar->start(count($this->source->getDocumentList()), LogManager::LOG_LEVEL_INFO);
$sourceDocuments = $this->source->getDocumentList();
$stage = 'run';
$processedDocuments = $this->progress->getProcessedEntities($this, $stage);
foreach (array_diff($sourceDocuments, $processedDocuments) as $sourceDocName) {
$this->progressBar->advance(LogManager::LOG_LEVEL_INFO);
$sourceDocument = $this->source->getDocument($sourceDocName);
$destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE);
if (!$destinationName) {
continue;
}
$destDocument = $this->destination->getDocument($destinationName);
$this->destination->clearDocument($destinationName);
$this->logger->debug('migrating', ['table' => $sourceDocName]);
$recordTransformer = $this->getRecordTransformer($sourceDocument, $destDocument);
$doCopy = $recordTransformer === null && $this->copyDirectly;
if ($doCopy && $this->isCopiedDirectly($sourceDocument, $destDocument)) {
$this->progressBar->start(1, LogManager::LOG_LEVEL_DEBUG);
} else {
$pageNumber = 0;
$this->progressBar->start(ceil($this->source->getRecordsCount($sourceDocName) / $this->source->getPageSize($sourceDocName)), LogManager::LOG_LEVEL_DEBUG);
while (!empty($items = $this->source->getRecords($sourceDocName, $pageNumber))) {
$pageNumber++;
$destinationRecords = $destDocument->getRecords();
foreach ($items as $data) {
if ($recordTransformer) {
/** @var Record $record */
$record = $this->recordFactory->create(['document' => $sourceDocument, 'data' => $data]);
/** @var Record $destRecord */
$destRecord = $this->recordFactory->create(['document' => $destDocument]);
$recordTransformer->transform($record, $destRecord);
} else {
$destRecord = $this->recordFactory->create(['document' => $destDocument, 'data' => $data]);
}
$destinationRecords->addRecord($destRecord);
}
$this->source->setLastLoadedRecord($sourceDocName, end($items));
$this->progressBar->advance(LogManager::LOG_LEVEL_DEBUG);
$fieldsUpdateOnDuplicate = $this->helper->getFieldsUpdateOnDuplicate($destinationName);
$this->destination->saveRecords($destinationName, $destinationRecords, $fieldsUpdateOnDuplicate);
}
}
$this->source->setLastLoadedRecord($sourceDocName, []);
$this->progress->addProcessedEntity($this, $stage, $sourceDocName);
$this->progressBar->finish(LogManager::LOG_LEVEL_DEBUG);
}
$this->progressBar->finish(LogManager::LOG_LEVEL_INFO);
return true;
}
/**
* {@inheritdoc}
*/
public function perform()
{
$sourceDocuments = array_keys($this->readerGroups->getGroup('source_documents'));
$this->progress->start(count($sourceDocuments), LogManager::LOG_LEVEL_INFO);
$this->helper->initEavEntityCollection('catalog_category_entity_varchar');
foreach ($sourceDocuments as $sourceDocName) {
$this->progress->advance(LogManager::LOG_LEVEL_INFO);
$sourceDocument = $this->source->getDocument($sourceDocName);
$destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE);
if (!$destinationName) {
continue;
}
$destDocument = $this->destination->getDocument($destinationName);
$this->destination->clearDocument($destinationName);
$this->logger->debug('migrating', ['table' => $sourceDocName]);
$recordTransformer = $this->recordTransformerFactory->create(['sourceDocument' => $sourceDocument, 'destDocument' => $destDocument, 'mapReader' => $this->map]);
$recordTransformer->init();
$this->progress->start(ceil($this->source->getRecordsCount($sourceDocName) / $this->source->getPageSize($sourceDocName)), LogManager::LOG_LEVEL_DEBUG);
$pageNumber = 0;
while (!empty($items = $this->source->getRecords($sourceDocName, $pageNumber))) {
$pageNumber++;
$destinationRecords = $destDocument->getRecords();
foreach ($items as $data) {
/** @var Record $record */
$record = $this->recordFactory->create(['document' => $sourceDocument, 'data' => $data]);
/** @var Record $destRecord */
$destRecord = $this->recordFactory->create(['document' => $destDocument]);
$recordTransformer->transform($record, $destRecord);
$destinationRecords->addRecord($destRecord);
$this->helper->updateAttributeData($data);
}
$this->source->setLastLoadedRecord($sourceDocName, end($items));
$this->progress->advance(LogManager::LOG_LEVEL_DEBUG);
$this->destination->saveRecords($destinationName, $destinationRecords);
}
$this->progress->finish(LogManager::LOG_LEVEL_DEBUG);
}
$this->helper->saveRecords();
$this->helper->updateEavAttributes();
$this->progress->finish(LogManager::LOG_LEVEL_INFO);
return true;
}
/**
* Migrate eav_attribute
* @return void
*/
protected function migrateAttributes()
{
$this->progress->advance();
$sourceDocName = 'eav_attribute';
$sourceDocument = $this->source->getDocument($sourceDocName);
$destinationDocument = $this->destination->getDocument($this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE));
$this->destination->backupDocument($destinationDocument->getName());
$sourceRecords = $this->source->getRecords($sourceDocName, 0, $this->source->getRecordsCount($sourceDocName));
foreach (array_keys($this->readerAttributes->getGroup('ignore')) as $attributeToClear) {
$sourceRecords = $this->clearIgnoredAttributes($sourceRecords, $attributeToClear);
}
$destinationRecords = $this->initialData->getAttributes('dest');
$recordsToSave = $destinationDocument->getRecords();
foreach ($sourceRecords as $sourceRecordData) {
/** @var Record $sourceRecord */
$sourceRecord = $this->factory->create(['document' => $sourceDocument, 'data' => $sourceRecordData]);
/** @var Record $destinationRecord */
$destinationRecord = $this->factory->create(['document' => $destinationDocument]);
$mappingValue = $this->getMappingValue($sourceRecord, ['entity_type_id', 'attribute_code']);
if (isset($destinationRecords[$mappingValue])) {
$destinationRecordData = $destinationRecords[$mappingValue];
unset($destinationRecords[$mappingValue]);
} else {
$destinationRecordData = array_fill_keys($destinationRecord->getFields(), null);
}
$destinationRecord->setData($destinationRecordData);
$this->helper->getRecordTransformer($sourceDocument, $destinationDocument)->transform($sourceRecord, $destinationRecord);
$recordsToSave->addRecord($destinationRecord);
}
foreach ($destinationRecords as $record) {
/** @var Record $destinationRecord */
$destinationRecord = $this->factory->create(['document' => $destinationDocument, 'data' => $record]);
$destinationRecord->setValue('attribute_id', null);
$recordsToSave->addRecord($destinationRecord);
}
$this->destination->clearDocument($destinationDocument->getName());
$this->saveRecords($destinationDocument, $recordsToSave);
$this->loadNewAttributes();
}
/**
* @return bool
* @throws \Migration\Exception
*/
public function perform()
{
$sourceDocuments = array_flip($this->source->getDocumentList());
foreach ($this->deltaDocuments as $documentName => $idKey) {
if (!$this->source->getDocument($documentName)) {
continue;
}
$deltaLogName = $this->source->getDeltaLogName($documentName);
if (!isset($sourceDocuments[$deltaLogName])) {
throw new \Migration\Exception(sprintf('Deltalog for %s is not installed', $documentName));
}
$destinationName = $this->mapReader->getDocumentMap($documentName, MapInterface::TYPE_SOURCE);
if (!$destinationName) {
continue;
}
if ($this->source->getRecordsCount($deltaLogName) == 0) {
continue;
}
$this->logger->debug(sprintf('%s has changes', $documentName));
$this->processDeletedRecords($documentName, $idKey, $destinationName);
$this->processChangedRecords($documentName, $idKey);
}
return true;
}
/**
* Get iterations count for step
*
* @return int
*/
protected function getIterationsCount()
{
return $this->source->getRecordsCount($this->tableName) + $this->source->getRecordsCount('catalog_category_entity_url_key') + $this->source->getRecordsCount('catalog_product_entity_url_key');
}
