/** * Builds eZDB connection to MDD database * Must be called before any other static method * @param eZINI $importINI * @return void */ public static function init(eZINI $importINI) { list($server, $port, $user, $pwd, $db, $usePersistentConnection, $databaseImplementation) = $importINI->variableMulti('DatabaseSettings', array( 'Server', 'Port', 'User', 'Password', 'Database', 'UsePersistentConnection', 'DatabaseImplementation' )); $databaseParameters = array( 'server' => $server, 'port' => $port, 'user' => $user, 'password' => $pwd, 'database' => $db, 'use_persistent_connection' => $usePersistentConnection ); self::$db = eZDB::instance($databaseImplementation, $databaseParameters, true); }
/** * Main method to process current row returned by getNextRow() method. * You may throw an exception if something goes wrong. It will be logged but won't break the import process * @param array $row Depending on your data format, can be DOMNode, SimpleXMLIterator, SimpleXMLElement, CSV row... * @throws Exception * @return void */ public function process( $row ) { $rootImport = $this->importINI->variable( 'XMLImportSettings', 'RootImport' ); $publisherInfos = $this->fileHandler->getPublisherInfosFromFolderName( $row['publisher'] ); if ( empty( $publisherInfos['default_language'] ) ) { $publisherInfos['default_language'] = 'eng-GB'; } $filePath = eZSys::rootDir() . "/$rootImport/{$this->fileHandler->rootImportFolder}/{$publisherInfos['path']}/{$row['file']}"; XMLImportMonitor::initFile( $row['file'], $publisherInfos['path'], $publisherInfos['id'], md5_file($filePath) ); try { if ( empty( $row['file'] ) || empty( $publisherInfos['path'] ) ) return; try { $xmlOptions = new SQLIXMLOptions( array( 'xml_path' => $filePath, 'xml_parser' => 'dom' ) ); $sqliXmlParser = new SQLIXMLParser( $xmlOptions ); $this->xmlParser = $sqliXmlParser->parse(); } catch ( SQLIXMLException $e ) { $this->triggerException( $row['file'], $publisherInfos['path'], $e, 4 ); } $xslProcess = XMLImportMapping::getByFieldName('xsl_process', $publisherInfos['path']); if ( $xslProcess !== false ) { if ( !isset($this->xmlTransformers[$publisherInfos['path']]) ) { try { $xslFolder = $this->importINI->variable('XMLImportSettings', 'XSLTFolder'); $xmslOptions = new SQLIXMLOptions( array( 'xml_path' => eZSys::rootDir() . '/extension/' . $xslFolder . '/' . $xslProcess, 'xml_parser' => 'dom' ) ); $sqliXmlTransformer = new SQLIXMLParser( $xmslOptions ); $path = $publisherInfos['path']; $this->xmlTransformers[$path] = $sqliXmlTransformer->parse(); } catch ( SQLIXMLException $e ) { $this->triggerException( $row['file'], $publisherInfos['path'], $e, 4 ); } } // Configure the transformer $proc = new XSLTProcessor; $proc->importStyleSheet($this->xmlTransformers[$publisherInfos['path']]); $this->xmlParser = $proc->transformToDoc($this->xmlParser); } try { $xmlFieldBuilder = new XMLFieldBuilder( $this->xmlParser, $this->importINI ); } catch ( RootNodeException $e ) { $this->triggerException( $row['file'], $publisherInfos['path'], $e, 4 ); } try { $xmlFieldMixer = new XMLFieldMixer( $xmlFieldBuilder, $this->xmlParser, $this->importINI, $publisherInfos, $this->rootImportFolder ); $fieldArray = $xmlFieldMixer->process(); } catch ( MandatoryException $e ) { $this->triggerException( $row['file'], $publisherInfos['path'], $e, 3 ); } try { XMLImportDataHandler::updateMdd( $fieldArray ); } catch ( XMLImportDBException $e ) { XMLImportDB::rollbackQuery(); $e = new Exception( $e->getMessage() . "\n" . 'SQL action failed - Transaction will be rollbacked - Error Message : ' . XMLImportDB::getErrorMessage() ); $this->triggerException( $row['file'], $publisherInfos['path'], $e, 5 ); } //Archive XML and blobed files $this->fileHandler->archiveFiles( $publisherInfos['path'], $row['file'], $xmlFieldMixer->blobedFiles ); } catch ( Exception $e ) { XMLImportMonitor::commitFile(); throw $e; } $this->fileHandler->dataSourcesCompleted++; XMLImportMonitor::commitFile(); }
/** * @param array $fields * @return string */ private static function implodeWithKeys($fields) { $string = ''; $first = true; foreach ($fields as $key => $field) { if (is_array($field)) { $field = implode("\n", $field); } if ($first) { $first = false; } else { $string .= '\', '; } $string .= '`' . $key . '` = \'' . XMLImportDB::escapeString($field); } return $string . '\''; }
$messageReport = ""; $csvMessages = array(); foreach ($results as $publisherInfo) { if ( !isset($since) ) { $query = "SELECT date,version,xml_file,error_messages,warning_messages from import_status where publisher_folder_id={$publisherInfo['id']} order by xml_file ASC, version DESC"; } else { $sinceTimestamp = time() - 60 * 60 * 24; $query = "SELECT date,version,xml_file,error_messages,warning_messages from import_status where publisher_folder_id={$publisherInfo['id']} and date >= $sinceTimestamp order by xml_file ASC, version DESC"; } $results = XMLImportDB::arrayQuery($query); if ( !$results ) { continue; } $currentXML = ""; $warningMessages = array(); $errorMessages = array(); $allMessages = array(); $publisherName = $publisherInfo['path']; $publisherCount = 0; $warningCount = 0; $errorCount = 0;
$contentQueries[] = "UPDATE import_status SET publisher_folder_id=(@temp:=publisher_folder_id), publisher_folder_id = new_publisher_folder_id, new_publisher_folder_id = @temp"; $contentQueries[] = "ALTER TABLE import_status CHANGE new_publisher_folder_id old_publisher_folder_id INT(11)"; $contentQueries[] = "UPDATE content_has_taxonomy SET taxonomy_id=(@temp:=taxonomy_id), taxonomy_id = new_taxonomy_id, taxonomy_id = @temp"; $contentQueries[] = "ALTER TABLE content_has_taxonomy CHANGE new_taxonomy_id old_mm_taxonomy_id INT(11)"; $db->query( "SET FOREIGN_KEY_CHECKS = 0" ); foreach( $queries as $q ) { $db->query( $q ); } foreach( $contentQueries as $q ) { XMLImportDB::query( $q ); } $db->commit(); $db->query( "SET FOREIGN_KEY_CHECKS = 1" ); $cli->output( "DONE." );
if ( $results ) { echo $index . ' : ' . count($results)."\n"; } else { echo "No results\n"; } } } else { foreach ( $deletes as $index => $delete ) { $resSelect = XMLImportDB::arrayQuery($selects[$index]); if ( $resSelect ) { $temp = array(); foreach ( $resSelect as $result ) { $temp[] = $result['result']; } $concat = '(' . implode(',', $temp) . ');'; $deleteQ = $delete . $concat; echo "$deleteQ\n"; }
/** * @param string $type * @param array $fieldArray * @param array $conditions * @throws XMLImportDBException * @return bool */ private static function updateDB( $type, $fieldArray, $conditions ) { $type = self::getAdjustedType( $fieldArray, $type ); if ( empty( $type ) || empty( $fieldArray ) ) return false; $table = 'content'; $sqlFields = self::buildSQLFields( $type, $table, $fieldArray ); $sqlFields['must_update'] = array( 'key' => 'must_update', 'value' => 1 ); $sqlFields = array_merge(self::getAllFieldsFromTable($table), $sqlFields); if( empty($sqlFields['parent_id']['value']) ) $sqlFields['parent_id']['value'] = null; $sql = sprintf( "UPDATE $table SET %s WHERE id=%s", self::implodeWithKeys( ", ", " = ", $sqlFields ), $conditions['content_id'] ); $result = XMLImportDB::query( $sql ); if ( !$result ) throw new XMLImportDBException( 'The content has not been successfuly updated' ); XMLImportMonitor::log( 'The content has been successfuly updated', 'info' ); $table = $type; $sqlFields = self::buildSQLFields( $type, $table, $fieldArray ); if ( !isset( $conditions['id'] ) ) { $sqlFields['content_id'] = array('key' => 'content_id','value' => $conditions['content_id']); $sql = "INSERT INTO %s SET %s"; $result = XMLImportDB::query( sprintf($sql, $table, self::implodeWithKeys( ", ", " = ", $sqlFields )) ); if ( $result ) { XMLImportMonitor::log( ucfirst( $type ) . " {$conditions['id']} successfuly created", 'info' ); $conditions['id'] = XMLImportDB::lastSerialID( $table, 'id' ); $conditions['language'] = $sqlFields['language']['value']; } } else { // merge $sqlFields with empty values for clear values which not present in xml $sqlFields = array_merge(self::getAllFieldsFromTable($table), $sqlFields); $sql = "UPDATE %s SET %s WHERE id=%s"; $result = XMLImportDB::query( sprintf($sql, $table, self::implodeWithKeys( ", ", " = ", $sqlFields ), $conditions['id']) ); if ( $result ) { XMLImportMonitor::log( ucfirst( $type ) . " {$conditions['id']} successfuly updated", 'info' ); $conditions['language'] = $sqlFields['language']['value']; } } if ( !$result ) throw new XMLImportDBException( ucfirst( $type ) . ' not successfuly updated' ); if ( $type != 'article' ) { $conditions['table_name'] = $table; $conditions['content_order'] = isset( $fieldArray['comp_art_order'] ) ? $fieldArray['comp_art_order']->calculatedValue : 0; } return $conditions; }
{ $filePart = explode('/', __FILE__); $i = 0; while ($filePart[$i] != 'extension' ) { unset ($filePart[$i]); $i++; } die('This is a eZexec script, please use php bin/php/ezexec.php -s site_admin ' . implode('/',$filePart) . "\n" ); } XMLImportDB::init(eZINI::instance('sqliimport.ini')); $dbEz = eZDB::instance(); $queryModel = "select id, path from publisher_folder"; $queryEz = "select id, path from mm_publisher_folder"; $resultsModel = XMLImportDB::arrayQuery($queryModel); $resultsEz = $dbEz->arrayQuery($queryEz); $mapping = array(); $stop = false; foreach ( $resultsModel as $resultModel ) { $mapping[$resultModel['path']] = array( 'modelId' => $resultModel['id'] ); } foreach ( $resultsEz as $resultEz ) { if ( !isset( $mapping[$resultEz['path']] )) { $mapping[$resultEz['path']] = array(); }
/** * Filters files toward retries * * @param array $list * @return array */ public function filterRetries ( $list ) { foreach (array_keys($list) as $key) { $file = $list[$key]; $fileSplit = explode('_', $file['file']); $fileSQL = $fileSplit[1] . '_' . $fileSplit[2]; $file = $list[$key]; $filePath = $this->rootImportFolder . '/' . $file['publisher'] . '/' . $file['file']; $publisherFolderId = $this->getPublisherIdFromFolderName($file['publisher']); $query = "SELECT file_md5 FROM %s WHERE `xml_file` = '%s' AND `publisher_folder_id` = '%s' ORDER BY version DESC"; $results = XMLImportDB::arrayQuery(sprintf($query, XMLImportMonitor::MONITORING_TABLE, $fileSQL, $publisherFolderId )); if ( $results ) { $fileMd5 = md5_file($filePath); if ( $fileMd5 && $fileMd5 == $results[0]['file_md5'] ) unset( $list[$key] ); } } return $list; }
function updateArticleImportStatus($importStatusId, $dryRun = false, eZCLI $cli = null, $statusMddEz = 0) { $query = "UPDATE import_status SET status_mdd_ez = '%s' WHERE id = '%s'"; $query = sprintf($query, $statusMddEz, $importStatusId); if ($dryRun) { $cli->warning($query); return; } XMLImportDB::query($query); }
elseif ( !isset($ids['modelId']) ) { $cli->output( "Publisher folder present in eZPublish but not in Model : {$path}" ); $stop = true; unset($mapping[$path]); } } if ( $stop && !$force ) { $cli->output( "Some publisher folders are not align : please align or delete if not necessary" ); eZExecution::cleanExit(); } foreach ( $mapping as $ids ) { $queryContent = "UPDATE content SET temp_id=%s WHERE publisher_folder_id=%s"; $res = XMLImportDB::query( sprintf($queryContent, $ids['ezId'], $ids['modelId']) ); echo sprintf($queryContent, $ids['ezId'], $ids['modelId']) . "\n"; $queryImportStatus = "UPDATE import_status SET temp_id=%s WHERE publisher_folder_id=%s"; echo sprintf($queryImportStatus, $ids['ezId'], $ids['modelId']) . "\n"; $res = XMLImportDB::query( sprintf($queryImportStatus, $ids['ezId'], $ids['modelId']) ); } $queryOverwriteContent = "UPDATE content SET publisher_folder_id=temp_id"; XMLImportDB::query($queryOverwriteContent); $queryOverwriteImportStatus = "UPDATE import_status SET publisher_folder_id=temp_id"; XMLImportDB::query($queryOverwriteImportStatus); eZExecution::cleanExit();
public static function run() { global $cli, $script; XMLImportDB::init(eZINI::instance('sqliimport.ini')); self::$dbEz = eZDB::instance(); self::$taxonomyCategoryIdentifiers = array(); self::$taxonomyCategoryMapping = array(); self::$taxonomyById = array(); self::$publisherFolders = array(); foreach( self::$dbEz->arrayQuery("SELECT t.id, t.code, ct.attribute_identifier FROM mm_taxonomy t, mm_taxonomy_category ct WHERE t.taxonomy_category_id = ct.id AND ct.attribute_identifier != 'conditions'") as $row ){ $taxonomyCategoryIdentifier = $row['attribute_identifier']; $taxonomyCode = $row['code']; self::$taxonomyById[$row['id'].''] = $taxonomyCode; $mapping = array_flip( self::$taxonomyCategoryIdentifiers ); $taxonomyCategoryIdentifierKey = null; if( !isset($mapping[$taxonomyCategoryIdentifier]) ) { self::$taxonomyCategoryIdentifiers[] = $taxonomyCategoryIdentifier; $taxonomyCategoryIdentifierKey = count(self::$taxonomyCategoryIdentifiers) - 1; } else { $taxonomyCategoryIdentifierKey = $mapping[$taxonomyCategoryIdentifier]; } self::$taxonomyCategoryMapping[$taxonomyCode] = $taxonomyCategoryIdentifierKey; } foreach( self::$dbEz->arrayQuery("SELECT id, path FROM mm_publisher_folder") as $row ) self::$publisherFolders[$row['id']] = $row['path']; $offset = 0; $step = 500; $continue = true; $count = 0; foreach( XMLImportDB::arrayQuery( "SELECT count(*) AS c FROM content" ) as $row ) $count = $row['c']; $cli->output( 'Updating taxonomies for content_ids - '.$count.' objects found' ); do { $sql = sprintf( "SELECT c.id, GROUP_CONCAT(a.language SEPARATOR '|') as lang, c.publisher_folder_id FROM content c, article a WHERE c.id = a.content_id AND c.id BETWEEN %d AND %d GROUP BY c.id ", $offset, ( $offset + $step - 1 ) ); self::$publishersAndLanguages = array(); foreach( XMLImportDB::arrayQuery($sql) as $row ) self::$publishersAndLanguages[$row['id']] = array( 'lang' => explode('|', $row['lang']), 'publisher' => $row['publisher_folder_id'] ); $sql = sprintf( "SELECT ct.content_id, ct.taxonomy_id FROM content_has_taxonomy ct WHERE ct.content_id BETWEEN %d AND %d ORDER BY ct.content_id ", $offset, ( $offset + $step - 1 ) ); $cli->output( str_pad( number_format( ($offset + $step ) / $count * 100, 2 ), 6, ' ', STR_PAD_LEFT )."% ".str_pad( $offset, 10, ' ', STR_PAD_LEFT).' > '.str_pad( $offset + $step - 1, 10, ' ', STR_PAD_LEFT ) ); self::$dbEz->begin(); $rows = XMLImportDB::arrayQuery( $sql ); $continue = count($rows); $objectId = null; $taxonomies = array(); foreach( $rows as $row ) { if( !is_null($objectId) && $objectId != $row['content_id'] ) { self::updateObject( $objectId, $taxonomies ); $objectId = $row['content_id']; $taxonomies = array( $row['taxonomy_id'] ); } else { $objectId = $row['content_id']; $taxonomies[] = $row['taxonomy_id']; } } if( $objectId ) self::updateObject( $objectId, $taxonomies ); $offset += $step; self::$dbEz->commit(); } while($continue); }
if ( $apply ) $cli->notice('Update...'); foreach ( $mappingOldIDWithKey as $id => $key ) { if ( !isset($mappingKeyWithNewID[$key]) ) { $cli->warning(sprintf('No corresponding between old (%s) and new ID for the key %s', $id, $key)); continue; } $newId = $mappingKeyWithNewID[$key]; $sql = sprintf('UPDATE content_has_taxonomy SET new_taxonomy_id = %s WHERE taxonomy_id = %s;', $newId, $id); if ( $apply ) { XMLImportDB::query($sql); } } if ( $apply && $tmpColumnExist ) { $qCli = new QuestionInteractiveCli(); $switchTable = $qCli->askQuestion('Switch actual column by the temporary column', array( 'Y' => 'New values replace the old and old value are maintained in a new column', 'N' => 'Do nothing : old values are maintained and the new value remain in the temporary column' )); if ( $switchTable == 'Y' ) {