/** * {@inheritdoc} */ public function perform() { $destinationDocument = $this->destination->getDocument(self::CONFIG_TABLE_NAME_DESTINATION); $recordsCountSource = $this->source->getRecordsCount(self::CONFIG_TABLE_NAME_SOURCE); $recordsCountDestination = $this->destination->getRecordsCount(self::CONFIG_TABLE_NAME_DESTINATION); $this->progress->start($recordsCountSource); $sourceRecords = $this->source->getRecords(self::CONFIG_TABLE_NAME_SOURCE, 0, $recordsCountSource); $destinationRecords = $this->destination->getRecords(self::CONFIG_TABLE_NAME_DESTINATION, 0, $recordsCountDestination); foreach ($sourceRecords as $sourceRecord) { $this->progress->advance(); if (!$this->readerSettings->isNodeIgnored($sourceRecord[self::CONFIG_FIELD_PATH])) { $sourceRecordPathMapped = $this->readerSettings->getNodeMap($sourceRecord[self::CONFIG_FIELD_PATH]); foreach ($destinationRecords as &$destinationRecord) { if ($destinationRecord[self::CONFIG_FIELD_SCOPE] == $sourceRecord[self::CONFIG_FIELD_SCOPE] && $destinationRecord[self::CONFIG_FIELD_SCOPE_ID] == $sourceRecord[self::CONFIG_FIELD_SCOPE_ID] && $destinationRecord[self::CONFIG_FIELD_PATH] == $sourceRecordPathMapped) { $record = $this->applyHandler($destinationDocument, $sourceRecord, $destinationRecord); $destinationRecord[self::CONFIG_FIELD_VALUE] = $record->getValue(self::CONFIG_FIELD_VALUE); continue 2; } } $record = $this->applyHandler($destinationDocument, $sourceRecord, []); $record->setValue(self::CONFIG_FIELD_PATH, $sourceRecordPathMapped); $destinationRecords[] = $record->getData(); } } foreach ($destinationRecords as &$destinationRecord) { unset($destinationRecord[self::CONFIG_FIELD_CONFIG_ID]); } $this->destination->clearDocument(self::CONFIG_TABLE_NAME_DESTINATION); $this->destination->saveRecords(self::CONFIG_TABLE_NAME_DESTINATION, $destinationRecords); $this->progress->finish(); return true; }
/** * @return bool */ public function perform() { // catalog_product_entity_tier_price should be migrated first to save same value_id as into magento1 $sourceDocuments = array_keys($this->helper->getSourceDocumentFields()); $this->progress->start(count($sourceDocuments), LogManager::LOG_LEVEL_INFO); $destinationName = $this->helper->getDestinationName(); $this->destination->clearDocument($destinationName); $destDocument = $this->destination->getDocument($destinationName); foreach ($sourceDocuments as $sourceDocName) { $pageNumber = 0; $this->logger->debug('migrating', ['table' => $sourceDocName]); $this->progress->start($this->source->getRecordsCount($sourceDocName), LogManager::LOG_LEVEL_DEBUG); while (!empty($items = $this->source->getRecords($sourceDocName, $pageNumber))) { $pageNumber++; $destinationRecords = $destDocument->getRecords(); foreach ($items as $recordData) { unset($recordData['value_id']); $this->progress->advance(LogManager::LOG_LEVEL_INFO); $this->progress->advance(LogManager::LOG_LEVEL_DEBUG); /** @var Record $destRecord */ $destRecord = $this->recordFactory->create(['document' => $destDocument, 'data' => $recordData]); $destinationRecords->addRecord($destRecord); } $this->destination->saveRecords($destinationName, $destinationRecords); } $this->progress->finish(LogManager::LOG_LEVEL_DEBUG); } $this->progress->finish(LogManager::LOG_LEVEL_INFO); return true; }
/** * @return void */ public function testGetRecordsCount() { $sourceCount = $this->source->getRecordsCount('table_with_data'); $destinationCount = $this->destination->getRecordsCount('table_without_data'); $this->assertEquals(7, $sourceCount); $this->assertEquals(0, $destinationCount); }
/** * {@inheritdoc} */ public function perform() { $this->progress->start(count($this->helper->getDocumentList())); foreach ($this->helper->getDocumentList() as $sourceName => $destinationName) { $this->progress->advance(); if ($this->source->getRecordsCount($sourceName) != $this->destination->getRecordsCount($destinationName)) { $this->errors[] = 'Mismatch of entities in the document: ' . $destinationName; } } $this->progress->finish(); return $this->checkForErrors(Logger::ERROR); }
/** * @return bool */ public function perform() { $this->progressBar->start($this->getIterationsCount()); foreach ($this->helper->getDocumentList() as $sourceDocumentName => $destinationDocumentName) { $this->progressBar->advance(); $sourceRecordsCount = $this->source->getRecordsCount($sourceDocumentName); $destinationRecordsCount = $this->destination->getRecordsCount($destinationDocumentName); if ($sourceRecordsCount != $destinationRecordsCount) { $this->errors[] = 'Mismatch of entities in the document: ' . $destinationDocumentName; } } $this->progressBar->finish(); return $this->checkForErrors(); }
/** * {@inheritdoc} */ public function perform() { $this->progress->start(count($this->helper->getSourceDocumentFields())); $sourceTotal = 0; $destinationTotal = $this->destination->getRecordsCount($this->helper->getDestinationName()); foreach (array_keys($this->helper->getSourceDocumentFields()) as $sourceName) { $sourceTotal += $this->source->getRecordsCount($sourceName); $this->progress->advance(); } if ($sourceTotal != $destinationTotal) { $this->errors[] = 'Mismatch of amount of entities in documents'; } $this->progress->finish(); return $this->checkForErrors(Logger::ERROR); }
/** * @return int */ protected function getIterationsCount() { $iterations = 0; $document = $this->helper->getDocumentList(); $iterations += $this->source->getRecordsCount($document['source']); return $iterations; }
/** * Get iterations count for step * * @return int */ protected function getIterationsCount() { $iterations = 0; foreach (array_keys($this->readerGroups->getGroup('source_documents')) as $document) { $iterations += $this->source->getRecordsCount($document); } return $iterations; }
/** * Get iterations count for step * * @return int */ protected function getIterationsCount() { $iterations = 0; foreach (array_keys($this->helper->getDocumentList()) as $document) { $iterations += $this->source->getRecordsCount($document); } return $iterations; }
/** * @param string $sourceDocName * @param string $destinationName * @return void */ protected function checkMapEntities($sourceDocName, $destinationName) { $sourceCount = $this->source->getRecordsCount($sourceDocName); $destinationCount = $this->destination->getRecordsCount($destinationName); if ($sourceCount != $destinationCount) { $this->errors[] = sprintf('Mismatch of entities in the document: %s', $destinationName); } }
/** * Volume check * * @return bool */ public function perform() { $sourceDocuments = array_keys($this->groups->getGroup('source_documents')); $this->progress->start(count($sourceDocuments)); foreach ($sourceDocuments as $sourceName) { $this->progress->advance(); $destinationName = $this->map->getDocumentMap($sourceName, MapInterface::TYPE_SOURCE); $sourceFields = $this->source->getDocument($sourceName)->getStructure()->getFields(); $destinationFields = $this->destination->getDocument($destinationName)->getStructure()->getFields(); if (!empty(array_diff_key($sourceFields, $destinationFields))) { $this->errors[] = 'Mismatch of fields in the document: ' . $destinationName; } if ($this->source->getRecordsCount($sourceName) != $this->destination->getRecordsCount($destinationName)) { $this->errors[] = 'Mismatch of entities in the document: ' . $destinationName; } } $this->progress->finish(); return $this->checkForErrors(); }
/** * @return bool */ public function perform() { $sourceDocuments = $this->source->getDocumentList(); $this->progressBar->start(count($sourceDocuments)); foreach ($sourceDocuments as $sourceDocName) { $this->progressBar->advance(); $destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE); if (!$destinationName) { continue; } $sourceCount = $this->source->getRecordsCount($sourceDocName); $destinationCount = $this->destination->getRecordsCount($destinationName); if ($sourceCount != $destinationCount) { $this->errors[] = 'Mismatch of entities in the document: ' . $destinationName; } } $this->progressBar->finish(); return $this->checkForErrors(); }
/** * Volume check * * @return bool */ protected function volume() { $result = true; $this->progress->start(1); $result &= $this->source->getRecordsCount(self::SOURCE) + $this->countCmsPageRewrites(true) == $this->destination->getRecordsCount(self::DESTINATION); if (!$result) { $this->logger->error('Mismatch of entities in the document: url_rewrite'); } $this->progress->advance(); $this->progress->finish(); return (bool) $result; }
/** * @param string $document * @return int */ public function getSourceRecordsCount($document) { if ($this->sourceDocuments[$document] == 'entity_id') { return $this->source->getRecordsCount($document); } $attributeType = $this->getAttributeType($document); /** @var Mysql $adapter */ $adapter = $this->source->getAdapter(); $query = $adapter->getSelect()->from(['et' => $this->source->addDocumentPrefix($document)], 'COUNT(*)')->where('et.attribute_id NOT IN (?)', array_keys($this->skipAttributes[$attributeType])); $count = $query->getAdapter()->fetchOne($query); return $count; }
/** * @return bool */ public function perform() { $sourceDocuments = array_keys($this->readerGroups->getGroup('source_documents')); $this->progress->start($this->getIterationsCount()); foreach ($sourceDocuments as $sourceDocName) { $this->progress->advance(); $destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE); if (!$destinationName) { continue; } $sourceCount = $this->source->getRecordsCount($sourceDocName); $destinationCount = $this->destination->getRecordsCount($destinationName); if ($sourceCount != $destinationCount) { $this->errors[] = 'Mismatch of entities in the document: ' . $destinationName; } } if (!$this->checkCleared(array_keys($this->readerGroups->getGroup('destination_documents_to_clear')))) { $this->errors[] = 'Log documents in the destination resource are not cleared'; } $this->progress->finish(); return $this->checkForErrors(); }
/** * {@inheritdoc} */ public function perform() { $updateData = $this->helper->getUpdateData(); $selectData = $this->helper->getSelectData(); $sourceDocuments = $this->readerGroups->getGroup('source_documents'); foreach ($sourceDocuments as $sourceDocName => $idKey) { if ($this->source->getRecordsCount($this->source->getDeltaLogName($sourceDocName)) == 0) { continue; } $items = $this->source->getChangedRecords($sourceDocName, $idKey, 0, true); if (empty($items)) { continue; } $this->logger->debug(sprintf('%s has changes', $sourceDocName)); if (!$this->eolOnce) { $this->eolOnce = true; echo PHP_EOL; } $gridIdKey = $updateData[$sourceDocName]['idKey']; $page = 1; do { $ids = []; foreach ($items as $data) { echo '.'; $ids[] = $data[$gridIdKey]; } foreach ($updateData[$sourceDocName]['methods'] as $method) { echo '.'; $destinationDocumentName = $selectData[$method]['destination']; $select = call_user_func_array([$this->data, $method], [$selectData[$method]['columns'], $ids]); $this->destination->getAdapter()->insertFromSelect($select, $this->destination->addDocumentPrefix($destinationDocumentName), [], \Magento\Framework\Db\Adapter\AdapterInterface::INSERT_ON_DUPLICATE); } $documentNameDelta = $this->source->getDeltaLogName($sourceDocName); $documentNameDelta = $this->source->addDocumentPrefix($documentNameDelta); $this->markRecordsProcessed($documentNameDelta, $idKey, $ids); } while (!empty($items = $this->source->getChangedRecords($sourceDocName, $idKey, $page++))); } return true; }
/** * @return bool */ public function perform() { $this->progressBar->start(count($this->source->getDocumentList()), LogManager::LOG_LEVEL_INFO); $sourceDocuments = $this->source->getDocumentList(); $stage = 'run'; $processedDocuments = $this->progress->getProcessedEntities($this, $stage); foreach (array_diff($sourceDocuments, $processedDocuments) as $sourceDocName) { $this->progressBar->advance(LogManager::LOG_LEVEL_INFO); $sourceDocument = $this->source->getDocument($sourceDocName); $destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE); if (!$destinationName) { continue; } $destDocument = $this->destination->getDocument($destinationName); $this->destination->clearDocument($destinationName); $this->logger->debug('migrating', ['table' => $sourceDocName]); $recordTransformer = $this->getRecordTransformer($sourceDocument, $destDocument); $doCopy = $recordTransformer === null && $this->copyDirectly; if ($doCopy && $this->isCopiedDirectly($sourceDocument, $destDocument)) { $this->progressBar->start(1, LogManager::LOG_LEVEL_DEBUG); } else { $pageNumber = 0; $this->progressBar->start(ceil($this->source->getRecordsCount($sourceDocName) / $this->source->getPageSize($sourceDocName)), LogManager::LOG_LEVEL_DEBUG); while (!empty($items = $this->source->getRecords($sourceDocName, $pageNumber))) { $pageNumber++; $destinationRecords = $destDocument->getRecords(); foreach ($items as $data) { if ($recordTransformer) { /** @var Record $record */ $record = $this->recordFactory->create(['document' => $sourceDocument, 'data' => $data]); /** @var Record $destRecord */ $destRecord = $this->recordFactory->create(['document' => $destDocument]); $recordTransformer->transform($record, $destRecord); } else { $destRecord = $this->recordFactory->create(['document' => $destDocument, 'data' => $data]); } $destinationRecords->addRecord($destRecord); } $this->source->setLastLoadedRecord($sourceDocName, end($items)); $this->progressBar->advance(LogManager::LOG_LEVEL_DEBUG); $fieldsUpdateOnDuplicate = $this->helper->getFieldsUpdateOnDuplicate($destinationName); $this->destination->saveRecords($destinationName, $destinationRecords, $fieldsUpdateOnDuplicate); } } $this->source->setLastLoadedRecord($sourceDocName, []); $this->progress->addProcessedEntity($this, $stage, $sourceDocName); $this->progressBar->finish(LogManager::LOG_LEVEL_DEBUG); } $this->progressBar->finish(LogManager::LOG_LEVEL_INFO); return true; }
/** * {@inheritdoc} */ public function perform() { $sourceDocuments = array_keys($this->readerGroups->getGroup('source_documents')); $this->progress->start(count($sourceDocuments), LogManager::LOG_LEVEL_INFO); $this->helper->initEavEntityCollection('catalog_category_entity_varchar'); foreach ($sourceDocuments as $sourceDocName) { $this->progress->advance(LogManager::LOG_LEVEL_INFO); $sourceDocument = $this->source->getDocument($sourceDocName); $destinationName = $this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE); if (!$destinationName) { continue; } $destDocument = $this->destination->getDocument($destinationName); $this->destination->clearDocument($destinationName); $this->logger->debug('migrating', ['table' => $sourceDocName]); $recordTransformer = $this->recordTransformerFactory->create(['sourceDocument' => $sourceDocument, 'destDocument' => $destDocument, 'mapReader' => $this->map]); $recordTransformer->init(); $this->progress->start(ceil($this->source->getRecordsCount($sourceDocName) / $this->source->getPageSize($sourceDocName)), LogManager::LOG_LEVEL_DEBUG); $pageNumber = 0; while (!empty($items = $this->source->getRecords($sourceDocName, $pageNumber))) { $pageNumber++; $destinationRecords = $destDocument->getRecords(); foreach ($items as $data) { /** @var Record $record */ $record = $this->recordFactory->create(['document' => $sourceDocument, 'data' => $data]); /** @var Record $destRecord */ $destRecord = $this->recordFactory->create(['document' => $destDocument]); $recordTransformer->transform($record, $destRecord); $destinationRecords->addRecord($destRecord); $this->helper->updateAttributeData($data); } $this->source->setLastLoadedRecord($sourceDocName, end($items)); $this->progress->advance(LogManager::LOG_LEVEL_DEBUG); $this->destination->saveRecords($destinationName, $destinationRecords); } $this->progress->finish(LogManager::LOG_LEVEL_DEBUG); } $this->helper->saveRecords(); $this->helper->updateEavAttributes(); $this->progress->finish(LogManager::LOG_LEVEL_INFO); return true; }
/** * Migrate eav_attribute * @return void */ protected function migrateAttributes() { $this->progress->advance(); $sourceDocName = 'eav_attribute'; $sourceDocument = $this->source->getDocument($sourceDocName); $destinationDocument = $this->destination->getDocument($this->map->getDocumentMap($sourceDocName, MapInterface::TYPE_SOURCE)); $this->destination->backupDocument($destinationDocument->getName()); $sourceRecords = $this->source->getRecords($sourceDocName, 0, $this->source->getRecordsCount($sourceDocName)); foreach (array_keys($this->readerAttributes->getGroup('ignore')) as $attributeToClear) { $sourceRecords = $this->clearIgnoredAttributes($sourceRecords, $attributeToClear); } $destinationRecords = $this->initialData->getAttributes('dest'); $recordsToSave = $destinationDocument->getRecords(); foreach ($sourceRecords as $sourceRecordData) { /** @var Record $sourceRecord */ $sourceRecord = $this->factory->create(['document' => $sourceDocument, 'data' => $sourceRecordData]); /** @var Record $destinationRecord */ $destinationRecord = $this->factory->create(['document' => $destinationDocument]); $mappingValue = $this->getMappingValue($sourceRecord, ['entity_type_id', 'attribute_code']); if (isset($destinationRecords[$mappingValue])) { $destinationRecordData = $destinationRecords[$mappingValue]; unset($destinationRecords[$mappingValue]); } else { $destinationRecordData = array_fill_keys($destinationRecord->getFields(), null); } $destinationRecord->setData($destinationRecordData); $this->helper->getRecordTransformer($sourceDocument, $destinationDocument)->transform($sourceRecord, $destinationRecord); $recordsToSave->addRecord($destinationRecord); } foreach ($destinationRecords as $record) { /** @var Record $destinationRecord */ $destinationRecord = $this->factory->create(['document' => $destinationDocument, 'data' => $record]); $destinationRecord->setValue('attribute_id', null); $recordsToSave->addRecord($destinationRecord); } $this->destination->clearDocument($destinationDocument->getName()); $this->saveRecords($destinationDocument, $recordsToSave); $this->loadNewAttributes(); }
/** * @return bool * @throws \Migration\Exception */ public function perform() { $sourceDocuments = array_flip($this->source->getDocumentList()); foreach ($this->deltaDocuments as $documentName => $idKey) { if (!$this->source->getDocument($documentName)) { continue; } $deltaLogName = $this->source->getDeltaLogName($documentName); if (!isset($sourceDocuments[$deltaLogName])) { throw new \Migration\Exception(sprintf('Deltalog for %s is not installed', $documentName)); } $destinationName = $this->mapReader->getDocumentMap($documentName, MapInterface::TYPE_SOURCE); if (!$destinationName) { continue; } if ($this->source->getRecordsCount($deltaLogName) == 0) { continue; } $this->logger->debug(sprintf('%s has changes', $documentName)); $this->processDeletedRecords($documentName, $idKey, $destinationName); $this->processChangedRecords($documentName, $idKey); } return true; }
/** * Get iterations count for step * * @return int */ protected function getIterationsCount() { return $this->source->getRecordsCount($this->tableName) + $this->source->getRecordsCount('catalog_category_entity_url_key') + $this->source->getRecordsCount('catalog_product_entity_url_key'); }