/**
* @return bool
*/
public function perform()
{
$deltaLogs = $this->groupsReader->getGroups();
$this->progress->start(count($deltaLogs, 1) - count($deltaLogs));
foreach ($deltaLogs as $deltaDocuments) {
foreach ($deltaDocuments as $documentName => $idKey) {
$this->progress->advance();
if ($this->source->getDocument($documentName)) {
$this->source->createDelta($documentName, $idKey);
}
}
}
$this->progress->finish();
return true;
}
/**
* @param string $documentName
* @param string $idKey
* @return void
*/
protected function processChangedRecords($documentName, $idKey)
{
$items = $this->source->getChangedRecords($documentName, $idKey);
if (empty($items)) {
return;
}
if (!$this->eolOnce) {
$this->eolOnce = true;
echo PHP_EOL;
}
$destinationName = $this->mapReader->getDocumentMap($documentName, MapInterface::TYPE_SOURCE);
$sourceDocument = $this->source->getDocument($documentName);
$destDocument = $this->destination->getDocument($destinationName);
$recordTransformer = $this->getRecordTransformer($sourceDocument, $destDocument);
do {
$destinationRecords = $destDocument->getRecords();
$ids = [];
foreach ($items as $data) {
echo '.';
$ids[] = $data[$idKey];
$this->transformData($data, $sourceDocument, $destDocument, $recordTransformer, $destinationRecords);
}
$this->destination->updateChangedRecords($destinationName, $destinationRecords);
$documentNameDelta = $this->source->getDeltaLogName($documentName);
$documentNameDelta = $this->source->addDocumentPrefix($documentNameDelta);
$this->markRecordsProcessed($documentNameDelta, $idKey, $ids);
} while (!empty($items = $this->source->getChangedRecords($documentName, $idKey)));
}
/**
* @return bool
*/
protected function data()
{
$this->progress->start(count($this->getDocumentList()));
$documents = $this->getDocumentList();
foreach ($documents as $sourceDocName => $destDocName) {
$this->progress->advance();
$sourceDocument = $this->source->getDocument($sourceDocName);
$destinationDocument = $this->destination->getDocument($destDocName);
$this->destination->clearDocument($destDocName);
$pageNumber = 0;
while (!empty($sourceRecords = $this->source->getRecords($sourceDocName, $pageNumber))) {
$pageNumber++;
$recordsToSave = $destinationDocument->getRecords();
foreach ($sourceRecords as $recordData) {
/** @var Resource\Record $destinationRecord */
$destinationRecord = $this->recordFactory->create(['document' => $destinationDocument]);
if ($this->haveEqualStructure($sourceDocument, $destinationDocument)) {
$destinationRecord->setData($recordData);
} else {
$destinationRecord = $this->transformRecord($destinationRecord, $recordData);
}
$recordsToSave->addRecord($destinationRecord);
}
$this->destination->saveRecords($destinationDocument->getName(), $recordsToSave);
}
}
$this->progress->finish();
return true;
}
/**
* Run step
*
* @return bool
*/
protected function data()
{
$this->progress->start($this->source->getRecordsCount(self::SOURCE));
$sourceDocument = $this->source->getDocument(self::SOURCE);
$destDocument = $this->destination->getDocument(self::DESTINATION);
$destProductCategory = $this->destination->getDocument(self::DESTINATION_PRODUCT_CATEGORY);
$this->destination->clearDocument(self::DESTINATION);
$this->destination->clearDocument(self::DESTINATION_PRODUCT_CATEGORY);
$pageNumber = 0;
while (!empty($bulk = $this->source->getRecords(self::SOURCE, $pageNumber))) {
$pageNumber++;
$destinationRecords = $destDocument->getRecords();
$destProductCategoryRecords = $destProductCategory->getRecords();
foreach ($bulk as $recordData) {
$this->progress->advance();
/** @var Record $record */
$record = $this->recordFactory->create(['document' => $sourceDocument, 'data' => $recordData]);
/** @var Record $destRecord */
$destRecord = $this->recordFactory->create(['document' => $destDocument]);
$this->transform($record, $destRecord);
if ($record->getValue('is_system') && $record->getValue('product_id') && $record->getValue('category_id')) {
$destProductCategoryRecord = $this->recordFactory->create(['document' => $destProductCategory]);
$destProductCategoryRecord->setValue('url_rewrite_id', $record->getValue('url_rewrite_id'));
$destProductCategoryRecord->setValue('category_id', $record->getValue('category_id'));
$destProductCategoryRecord->setValue('product_id', $record->getValue('product_id'));
$destProductCategoryRecords->addRecord($destProductCategoryRecord);
}
$destinationRecords->addRecord($destRecord);
}
$this->destination->saveRecords(self::DESTINATION, $destinationRecords);
$this->destination->saveRecords(self::DESTINATION_PRODUCT_CATEGORY, $destProductCategoryRecords);
}
$this->progress->finish();
return true;
}
/**
* @return bool
*/
public function perform()
{
$this->progressBar->start(count($this->source->getDocumentList()), LogManager::LOG_LEVEL_INFO);
$sourceDocuments = $this->source->getDocumentList();
$stage = 'run';
$processedDocuments = $this->progress->getProcessedEntities($this, $stage);
foreach (array_diff($sourceDocuments, $processedDocuments) as $sourceDocName) {
$this->progressBar->advance(LogManager::LOG_LEVEL_INFO);
$sourceDocument = $this->source->getDocument($sourceDocName);
$destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE);
if (!$destinationName) {
continue;
}
$destDocument = $this->destination->getDocument($destinationName);
$this->destination->clearDocument($destinationName);
$this->logger->debug('migrating', ['table' => $sourceDocName]);
$recordTransformer = $this->getRecordTransformer($sourceDocument, $destDocument);
$doCopy = $recordTransformer === null && $this->copyDirectly;
if ($doCopy && $this->isCopiedDirectly($sourceDocument, $destDocument)) {
$this->progressBar->start(1, LogManager::LOG_LEVEL_DEBUG);
} else {
$pageNumber = 0;
$this->progressBar->start(ceil($this->source->getRecordsCount($sourceDocName) / $this->source->getPageSize($sourceDocName)), LogManager::LOG_LEVEL_DEBUG);
while (!empty($items = $this->source->getRecords($sourceDocName, $pageNumber))) {
$pageNumber++;
$destinationRecords = $destDocument->getRecords();
foreach ($items as $data) {
if ($recordTransformer) {
/** @var Record $record */
$record = $this->recordFactory->create(['document' => $sourceDocument, 'data' => $data]);
/** @var Record $destRecord */
$destRecord = $this->recordFactory->create(['document' => $destDocument]);
$recordTransformer->transform($record, $destRecord);
} else {
$destRecord = $this->recordFactory->create(['document' => $destDocument, 'data' => $data]);
}
$destinationRecords->addRecord($destRecord);
}
$this->source->setLastLoadedRecord($sourceDocName, end($items));
$this->progressBar->advance(LogManager::LOG_LEVEL_DEBUG);
$this->destination->saveRecords($destinationName, $destinationRecords);
}
}
$this->source->setLastLoadedRecord($sourceDocName, []);
$this->progress->addProcessedEntity($this, $stage, $sourceDocName);
$this->progressBar->finish(LogManager::LOG_LEVEL_DEBUG);
}
$this->progressBar->finish(LogManager::LOG_LEVEL_INFO);
return true;
}
/**
* Migrate EAV tables which in result must have all unique records from both source and destination documents
* @return void
*/
protected function migrateMappedTables()
{
$documents = $this->readerGroups->getGroup('mapped_documents');
foreach ($documents as $documentName => $mappingFields) {
$this->progress->advance();
$sourceDocument = $this->source->getDocument($documentName);
$destinationDocument = $this->destination->getDocument($this->map->getDocumentMap($documentName, MapInterface::TYPE_SOURCE));
$this->destination->backupDocument($destinationDocument->getName());
$mappingFields = explode(',', $mappingFields);
$destinationRecords = $this->helper->getDestinationRecords($documentName, $mappingFields);
$recordsToSave = $destinationDocument->getRecords();
foreach ($this->helper->getSourceRecords($documentName) as $recordData) {
/** @var Record $sourceRecord */
$sourceRecord = $this->factory->create(['document' => $sourceDocument, 'data' => $recordData]);
/** @var Record $destinationRecord */
$destinationRecord = $this->factory->create(['document' => $destinationDocument]);
$mappingValue = $this->getMappingValue($sourceRecord, $mappingFields);
if (isset($destinationRecords[$mappingValue])) {
$destinationRecordData = $destinationRecords[$mappingValue];
unset($destinationRecords[$mappingValue]);
} else {
$destinationRecordData = array_fill_keys($destinationRecord->getFields(), null);
}
$destinationRecord->setData($destinationRecordData);
$this->helper->getRecordTransformer($sourceDocument, $destinationDocument)->transform($sourceRecord, $destinationRecord);
if ($documentName == 'eav_entity_type') {
$oldAttributeSetValue = $destinationRecord->getValue('default_attribute_set_id');
if (isset($this->destAttributeSetsOldNewMap[$oldAttributeSetValue])) {
$destinationRecord->setValue('default_attribute_set_id', $this->destAttributeSetsOldNewMap[$oldAttributeSetValue]);
}
}
$recordsToSave->addRecord($destinationRecord);
}
$this->destination->clearDocument($destinationDocument->getName());
$this->saveRecords($destinationDocument, $recordsToSave);
$recordsToSave = $destinationDocument->getRecords();
if ($mappingFields) {
foreach ($destinationRecords as $record) {
$destinationRecord = $this->factory->create(['document' => $destinationDocument, 'data' => $record]);
if (isset($record['attribute_id']) && isset($this->destAttributeOldNewMap[$record['attribute_id']])) {
$destinationRecord->setValue('attribute_id', $this->destAttributeOldNewMap[$record['attribute_id']]);
}
$recordsToSave->addRecord($destinationRecord);
}
}
$this->saveRecords($destinationDocument, $recordsToSave);
}
}
/**
* @return bool
*/
public function perform()
{
$this->progressBar->start(count($this->source->getDocumentList()), LogManager::LOG_LEVEL_INFO);
$sourceDocuments = $this->source->getDocumentList();
// TODO: during steps refactoring MAGETWO-35749 stage will be removed
$stage = 'run';
$processedDocuments = $this->progress->getProcessedEntities($this, $stage);
foreach ($sourceDocuments as $sourceDocName) {
$this->progressBar->advance(LogManager::LOG_LEVEL_INFO);
if (in_array($sourceDocName, $processedDocuments)) {
continue;
}
$sourceDocument = $this->source->getDocument($sourceDocName);
$destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE);
if (!$destinationName) {
continue;
}
$destDocument = $this->destination->getDocument($destinationName);
$this->destination->clearDocument($destinationName);
$recordTransformer = $this->getRecordTransformer($sourceDocument, $destDocument);
$pageNumber = 0;
$this->logger->debug('migrating', ['table' => $sourceDocName]);
$this->progressBar->start($this->source->getRecordsCount($sourceDocName), LogManager::LOG_LEVEL_DEBUG);
while (!empty($items = $this->source->getRecords($sourceDocName, $pageNumber))) {
$pageNumber++;
$destinationRecords = $destDocument->getRecords();
foreach ($items as $data) {
$this->progressBar->advance(LogManager::LOG_LEVEL_DEBUG);
if ($recordTransformer) {
/** @var Record $record */
$record = $this->recordFactory->create(['document' => $sourceDocument, 'data' => $data]);
/** @var Record $destRecord */
$destRecord = $this->recordFactory->create(['document' => $destDocument]);
$recordTransformer->transform($record, $destRecord);
} else {
$destRecord = $this->recordFactory->create(['document' => $destDocument, 'data' => $data]);
}
$destinationRecords->addRecord($destRecord);
}
$this->destination->saveRecords($destinationName, $destinationRecords);
}
$this->progress->addProcessedEntity($this, $stage, $sourceDocName);
$this->progressBar->finish(LogManager::LOG_LEVEL_DEBUG);
}
$this->progressBar->finish(LogManager::LOG_LEVEL_INFO);
return true;
}
/**
* @return bool
*/
public function perform()
{
$this->progress->start($this->getIterationsCount(), LogManager::LOG_LEVEL_INFO);
$sourceDocuments = array_keys($this->readerGroups->getGroup('source_documents'));
foreach ($sourceDocuments as $sourceDocName) {
$sourceDocument = $this->source->getDocument($sourceDocName);
$destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE);
if (!$destinationName) {
continue;
}
$destDocument = $this->destination->getDocument($destinationName);
$this->destination->clearDocument($destinationName);
/** @var \Migration\RecordTransformer $recordTransformer */
$recordTransformer = $this->recordTransformerFactory->create(['sourceDocument' => $sourceDocument, 'destDocument' => $destDocument, 'mapReader' => $this->map]);
$recordTransformer->init();
$attributeType = $this->helper->getAttributeType($sourceDocName);
$pageNumber = 0;
$this->logger->debug('migrating', ['table' => $sourceDocName]);
$this->progress->start(ceil($this->source->getRecordsCount($sourceDocName) / $this->source->getPageSize($sourceDocName)), LogManager::LOG_LEVEL_DEBUG);
while (!empty($bulk = $this->source->getRecords($sourceDocName, $pageNumber))) {
$pageNumber++;
$destinationRecords = $destDocument->getRecords();
foreach ($bulk as $recordData) {
$this->source->setLastLoadedRecord($sourceDocName, $recordData);
if ($this->helper->isSkipRecord($attributeType, $sourceDocName, $recordData)) {
continue;
}
/** @var Record $record */
$record = $this->recordFactory->create(['document' => $sourceDocument, 'data' => $recordData]);
/** @var Record $destRecord */
$destRecord = $this->recordFactory->create(['document' => $destDocument]);
$recordTransformer->transform($record, $destRecord);
$destinationRecords->addRecord($destRecord);
}
$this->progress->advance(LogManager::LOG_LEVEL_INFO);
$this->progress->advance(LogManager::LOG_LEVEL_DEBUG);
$this->helper->updateAttributeData($attributeType, $sourceDocName, $destinationRecords);
$this->destination->saveRecords($destinationName, $destinationRecords);
}
$this->progress->finish(LogManager::LOG_LEVEL_DEBUG);
}
$this->helper->updateEavAttributes();
$this->progress->finish(LogManager::LOG_LEVEL_INFO);
return true;
}
/**
* Data migration
*
* @return bool
* @throws \Migration\Exception
*/
protected function data()
{
$this->getRewritesSelect();
$this->progress->start($this->getIterationsCount());
$sourceDocument = $this->source->getDocument($this->tableName);
$destinationDocument = $this->destination->getDocument('url_rewrite');
$destProductCategory = $this->destination->getDocument('catalog_url_rewrite_product_category');
$duplicates = $this->getDuplicatesList();
if (!empty($duplicates) && !empty($this->configReader->getOption('auto_resolve_urlrewrite_duplicates')) && empty($this->duplicateIndex)) {
foreach ($duplicates as $row) {
$this->duplicateIndex[$row['request_path']][] = $row;
}
}
$pageNumber = 0;
while (!empty($data = $this->source->getRecords($sourceDocument->getName(), $pageNumber))) {
$pageNumber++;
$records = $this->recordCollectionFactory->create();
$destProductCategoryRecords = $destProductCategory->getRecords();
foreach ($data as $row) {
$this->progress->advance();
$records->addRecord($this->recordFactory->create(['data' => $row]));
$productCategoryRecord = $this->getProductCategoryRecord($destProductCategory, $row);
if ($productCategoryRecord) {
$destProductCategoryRecords->addRecord($productCategoryRecord);
}
}
$destinationRecords = $destinationDocument->getRecords();
$this->migrateRewrites($records, $destinationRecords);
$this->destination->saveRecords($destinationDocument->getName(), $destinationRecords);
$this->destination->saveRecords($destProductCategory->getName(), $destProductCategoryRecords);
}
$this->copyEavData('catalog_category_entity_url_key', 'catalog_category_entity_varchar', 'category');
$this->copyEavData('catalog_product_entity_url_key', 'catalog_product_entity_varchar', 'product');
$this->progress->finish();
return true;
}
/**
* @param string $dataTable
* @param \Migration\Resource\Source $resource
*/
protected function checkDeltaLogTable($dataTable, $resource)
{
$deltaLogTableName = $resource->getDeltaLogName($dataTable);
$deltaLogTable = $resource->getDocument($deltaLogTableName);
$this->assertEquals($deltaLogTableName, $deltaLogTable->getName());
}
public function testGetFields()
{
$sourceStruct = $this->source->getDocument('table_with_data')->getStructure()->getFields();
$destStruct = $this->destination->getDocument('table_without_data')->getStructure()->getFields();
$this->assertEquals(array_keys($sourceStruct), array_keys($destStruct));
}