/**
* Builds eZDB connection to MDD database
* Must be called before any other static method
* @param eZINI $importINI
* @return void
*/
public static function init(eZINI $importINI)
{
list($server, $port, $user, $pwd, $db, $usePersistentConnection, $databaseImplementation) = $importINI->variableMulti('DatabaseSettings', array(
'Server',
'Port',
'User',
'Password',
'Database',
'UsePersistentConnection',
'DatabaseImplementation'
));
$databaseParameters = array(
'server' => $server,
'port' => $port,
'user' => $user,
'password' => $pwd,
'database' => $db,
'use_persistent_connection' => $usePersistentConnection
);
self::$db = eZDB::instance($databaseImplementation, $databaseParameters, true);
}
/**
* Main method to process current row returned by getNextRow() method.
* You may throw an exception if something goes wrong. It will be logged but won't break the import process
* @param array $row Depending on your data format, can be DOMNode, SimpleXMLIterator, SimpleXMLElement, CSV row...
* @throws Exception
* @return void
*/
public function process( $row )
{
$rootImport = $this->importINI->variable( 'XMLImportSettings', 'RootImport' );
$publisherInfos = $this->fileHandler->getPublisherInfosFromFolderName( $row['publisher'] );
if ( empty( $publisherInfos['default_language'] ) ) {
$publisherInfos['default_language'] = 'eng-GB';
}
$filePath = eZSys::rootDir() . "/$rootImport/{$this->fileHandler->rootImportFolder}/{$publisherInfos['path']}/{$row['file']}";
XMLImportMonitor::initFile( $row['file'], $publisherInfos['path'], $publisherInfos['id'], md5_file($filePath) );
try
{
if ( empty( $row['file'] ) || empty( $publisherInfos['path'] ) )
return;
try
{
$xmlOptions = new SQLIXMLOptions( array(
'xml_path' => $filePath,
'xml_parser' => 'dom'
) );
$sqliXmlParser = new SQLIXMLParser( $xmlOptions );
$this->xmlParser = $sqliXmlParser->parse();
}
catch ( SQLIXMLException $e )
{
$this->triggerException( $row['file'], $publisherInfos['path'], $e, 4 );
}
$xslProcess = XMLImportMapping::getByFieldName('xsl_process', $publisherInfos['path']);
if ( $xslProcess !== false )
{
if ( !isset($this->xmlTransformers[$publisherInfos['path']]) )
{
try
{
$xslFolder = $this->importINI->variable('XMLImportSettings', 'XSLTFolder');
$xmslOptions = new SQLIXMLOptions( array(
'xml_path' => eZSys::rootDir() . '/extension/' . $xslFolder . '/' . $xslProcess,
'xml_parser' => 'dom'
) );
$sqliXmlTransformer = new SQLIXMLParser( $xmslOptions );
$path = $publisherInfos['path'];
$this->xmlTransformers[$path] = $sqliXmlTransformer->parse();
}
catch ( SQLIXMLException $e )
{
$this->triggerException( $row['file'], $publisherInfos['path'], $e, 4 );
}
}
// Configure the transformer
$proc = new XSLTProcessor;
$proc->importStyleSheet($this->xmlTransformers[$publisherInfos['path']]);
$this->xmlParser = $proc->transformToDoc($this->xmlParser);
}
try
{
$xmlFieldBuilder = new XMLFieldBuilder( $this->xmlParser, $this->importINI );
}
catch ( RootNodeException $e )
{
$this->triggerException( $row['file'], $publisherInfos['path'], $e, 4 );
}
try
{
$xmlFieldMixer = new XMLFieldMixer( $xmlFieldBuilder, $this->xmlParser, $this->importINI, $publisherInfos, $this->rootImportFolder );
$fieldArray = $xmlFieldMixer->process();
}
catch ( MandatoryException $e )
{
$this->triggerException( $row['file'], $publisherInfos['path'], $e, 3 );
}
try
{
XMLImportDataHandler::updateMdd( $fieldArray );
}
catch ( XMLImportDBException $e )
{
XMLImportDB::rollbackQuery();
$e = new Exception( $e->getMessage() . "\n" . 'SQL action failed - Transaction will be rollbacked - Error Message : ' . XMLImportDB::getErrorMessage() );
$this->triggerException( $row['file'], $publisherInfos['path'], $e, 5 );
}
//Archive XML and blobed files
$this->fileHandler->archiveFiles( $publisherInfos['path'], $row['file'], $xmlFieldMixer->blobedFiles );
}
catch ( Exception $e )
{
XMLImportMonitor::commitFile();
throw $e;
}
$this->fileHandler->dataSourcesCompleted++;
XMLImportMonitor::commitFile();
}
/**
* @param array $fields
* @return string
*/
private static function implodeWithKeys($fields)
{
$string = '';
$first = true;
foreach ($fields as $key => $field)
{
if (is_array($field))
{
$field = implode("\n", $field);
}
if ($first)
{
$first = false;
}
else
{
$string .= '\', ';
}
$string .= '`' . $key . '` = \'' . XMLImportDB::escapeString($field);
}
return $string . '\'';
}
$messageReport = "";
$csvMessages = array();
foreach ($results as $publisherInfo)
{
if ( !isset($since) )
{
$query = "SELECT date,version,xml_file,error_messages,warning_messages from import_status where publisher_folder_id={$publisherInfo['id']} order by xml_file ASC, version DESC";
}
else
{
$sinceTimestamp = time() - 60 * 60 * 24;
$query = "SELECT date,version,xml_file,error_messages,warning_messages from import_status where publisher_folder_id={$publisherInfo['id']} and date >= $sinceTimestamp order by xml_file ASC, version DESC";
}
$results = XMLImportDB::arrayQuery($query);
if ( !$results )
{
continue;
}
$currentXML = "";
$warningMessages = array();
$errorMessages = array();
$allMessages = array();
$publisherName = $publisherInfo['path'];
$publisherCount = 0;
$warningCount = 0;
$errorCount = 0;
$contentQueries[] = "UPDATE import_status
SET
publisher_folder_id=(@temp:=publisher_folder_id),
publisher_folder_id = new_publisher_folder_id,
new_publisher_folder_id = @temp";
$contentQueries[] = "ALTER TABLE import_status CHANGE new_publisher_folder_id old_publisher_folder_id INT(11)";
$contentQueries[] = "UPDATE content_has_taxonomy
SET
taxonomy_id=(@temp:=taxonomy_id),
taxonomy_id = new_taxonomy_id,
taxonomy_id = @temp";
$contentQueries[] = "ALTER TABLE content_has_taxonomy CHANGE new_taxonomy_id old_mm_taxonomy_id INT(11)";
$db->query( "SET FOREIGN_KEY_CHECKS = 0" );
foreach( $queries as $q )
{
$db->query( $q );
}
foreach( $contentQueries as $q )
{
XMLImportDB::query( $q );
}
$db->commit();
$db->query( "SET FOREIGN_KEY_CHECKS = 1" );
$cli->output( "DONE." );
if ( $results )
{
echo $index . ' : ' . count($results)."\n";
}
else
{
echo "No results\n";
}
}
}
else
{
foreach ( $deletes as $index => $delete )
{
$resSelect = XMLImportDB::arrayQuery($selects[$index]);
if ( $resSelect )
{
$temp = array();
foreach ( $resSelect as $result )
{
$temp[] = $result['result'];
}
$concat = '(' . implode(',', $temp) . ');';
$deleteQ = $delete . $concat;
echo "$deleteQ\n";
}
/**
* @param string $type
* @param array $fieldArray
* @param array $conditions
* @throws XMLImportDBException
* @return bool
*/
private static function updateDB( $type, $fieldArray, $conditions )
{
$type = self::getAdjustedType( $fieldArray, $type );
if ( empty( $type ) || empty( $fieldArray ) )
return false;
$table = 'content';
$sqlFields = self::buildSQLFields( $type, $table, $fieldArray );
$sqlFields['must_update'] = array( 'key' => 'must_update', 'value' => 1 );
$sqlFields = array_merge(self::getAllFieldsFromTable($table), $sqlFields);
if( empty($sqlFields['parent_id']['value']) )
$sqlFields['parent_id']['value'] = null;
$sql = sprintf( "UPDATE $table SET %s WHERE id=%s", self::implodeWithKeys( ", ", " = ", $sqlFields ), $conditions['content_id'] );
$result = XMLImportDB::query( $sql );
if ( !$result )
throw new XMLImportDBException( 'The content has not been successfuly updated' );
XMLImportMonitor::log( 'The content has been successfuly updated', 'info' );
$table = $type;
$sqlFields = self::buildSQLFields( $type, $table, $fieldArray );
if ( !isset( $conditions['id'] ) )
{
$sqlFields['content_id'] = array('key' => 'content_id','value' => $conditions['content_id']);
$sql = "INSERT INTO %s SET %s";
$result = XMLImportDB::query( sprintf($sql, $table, self::implodeWithKeys( ", ", " = ", $sqlFields )) );
if ( $result )
{
XMLImportMonitor::log( ucfirst( $type ) . " {$conditions['id']} successfuly created", 'info' );
$conditions['id'] = XMLImportDB::lastSerialID( $table, 'id' );
$conditions['language'] = $sqlFields['language']['value'];
}
}
else
{
// merge $sqlFields with empty values for clear values which not present in xml
$sqlFields = array_merge(self::getAllFieldsFromTable($table), $sqlFields);
$sql = "UPDATE %s SET %s WHERE id=%s";
$result = XMLImportDB::query( sprintf($sql, $table, self::implodeWithKeys( ", ", " = ", $sqlFields ), $conditions['id']) );
if ( $result )
{
XMLImportMonitor::log( ucfirst( $type ) . " {$conditions['id']} successfuly updated", 'info' );
$conditions['language'] = $sqlFields['language']['value'];
}
}
if ( !$result )
throw new XMLImportDBException( ucfirst( $type ) . ' not successfuly updated' );
if ( $type != 'article' )
{
$conditions['table_name'] = $table;
$conditions['content_order'] = isset( $fieldArray['comp_art_order'] ) ? $fieldArray['comp_art_order']->calculatedValue : 0;
}
return $conditions;
}
{
$filePart = explode('/', __FILE__);
$i = 0;
while ($filePart[$i] != 'extension' )
{
unset ($filePart[$i]);
$i++;
}
die('This is a eZexec script, please use php bin/php/ezexec.php -s site_admin ' . implode('/',$filePart) . "\n" );
}
XMLImportDB::init(eZINI::instance('sqliimport.ini'));
$dbEz = eZDB::instance();
$queryModel = "select id, path from publisher_folder";
$queryEz = "select id, path from mm_publisher_folder";
$resultsModel = XMLImportDB::arrayQuery($queryModel);
$resultsEz = $dbEz->arrayQuery($queryEz);
$mapping = array();
$stop = false;
foreach ( $resultsModel as $resultModel )
{
$mapping[$resultModel['path']] = array( 'modelId' => $resultModel['id'] );
}
foreach ( $resultsEz as $resultEz )
{
if ( !isset( $mapping[$resultEz['path']] ))
{
$mapping[$resultEz['path']] = array();
}
/**
* Filters files toward retries
*
* @param array $list
* @return array
*/
public function filterRetries ( $list )
{
foreach (array_keys($list) as $key)
{
$file = $list[$key];
$fileSplit = explode('_', $file['file']);
$fileSQL = $fileSplit[1] . '_' . $fileSplit[2];
$file = $list[$key];
$filePath = $this->rootImportFolder . '/' . $file['publisher'] . '/' . $file['file'];
$publisherFolderId = $this->getPublisherIdFromFolderName($file['publisher']);
$query = "SELECT file_md5 FROM %s WHERE `xml_file` = '%s' AND `publisher_folder_id` = '%s' ORDER BY version DESC";
$results = XMLImportDB::arrayQuery(sprintf($query, XMLImportMonitor::MONITORING_TABLE, $fileSQL, $publisherFolderId ));
if ( $results )
{
$fileMd5 = md5_file($filePath);
if ( $fileMd5 && $fileMd5 == $results[0]['file_md5'] )
unset( $list[$key] );
}
}
return $list;
}
function updateArticleImportStatus($importStatusId, $dryRun = false, eZCLI $cli = null, $statusMddEz = 0)
{
$query = "UPDATE import_status SET status_mdd_ez = '%s' WHERE id = '%s'";
$query = sprintf($query, $statusMddEz, $importStatusId);
if ($dryRun)
{
$cli->warning($query);
return;
}
XMLImportDB::query($query);
}
elseif ( !isset($ids['modelId']) )
{
$cli->output( "Publisher folder present in eZPublish but not in Model : {$path}" );
$stop = true;
unset($mapping[$path]);
}
}
if ( $stop && !$force )
{
$cli->output( "Some publisher folders are not align : please align or delete if not necessary" );
eZExecution::cleanExit();
}
foreach ( $mapping as $ids )
{
$queryContent = "UPDATE content SET temp_id=%s WHERE publisher_folder_id=%s";
$res = XMLImportDB::query( sprintf($queryContent, $ids['ezId'], $ids['modelId']) );
echo sprintf($queryContent, $ids['ezId'], $ids['modelId']) . "\n";
$queryImportStatus = "UPDATE import_status SET temp_id=%s WHERE publisher_folder_id=%s";
echo sprintf($queryImportStatus, $ids['ezId'], $ids['modelId']) . "\n";
$res = XMLImportDB::query( sprintf($queryImportStatus, $ids['ezId'], $ids['modelId']) );
}
$queryOverwriteContent = "UPDATE content SET publisher_folder_id=temp_id";
XMLImportDB::query($queryOverwriteContent);
$queryOverwriteImportStatus = "UPDATE import_status SET publisher_folder_id=temp_id";
XMLImportDB::query($queryOverwriteImportStatus);
eZExecution::cleanExit();
public static function run()
{
global $cli, $script;
XMLImportDB::init(eZINI::instance('sqliimport.ini'));
self::$dbEz = eZDB::instance();
self::$taxonomyCategoryIdentifiers = array();
self::$taxonomyCategoryMapping = array();
self::$taxonomyById = array();
self::$publisherFolders = array();
foreach( self::$dbEz->arrayQuery("SELECT
t.id,
t.code,
ct.attribute_identifier
FROM
mm_taxonomy t,
mm_taxonomy_category ct
WHERE
t.taxonomy_category_id = ct.id AND
ct.attribute_identifier != 'conditions'") as $row
){
$taxonomyCategoryIdentifier = $row['attribute_identifier'];
$taxonomyCode = $row['code'];
self::$taxonomyById[$row['id'].''] = $taxonomyCode;
$mapping = array_flip( self::$taxonomyCategoryIdentifiers );
$taxonomyCategoryIdentifierKey = null;
if( !isset($mapping[$taxonomyCategoryIdentifier]) )
{
self::$taxonomyCategoryIdentifiers[] = $taxonomyCategoryIdentifier;
$taxonomyCategoryIdentifierKey = count(self::$taxonomyCategoryIdentifiers) - 1;
}
else
{
$taxonomyCategoryIdentifierKey = $mapping[$taxonomyCategoryIdentifier];
}
self::$taxonomyCategoryMapping[$taxonomyCode] = $taxonomyCategoryIdentifierKey;
}
foreach( self::$dbEz->arrayQuery("SELECT id, path
FROM mm_publisher_folder") as $row )
self::$publisherFolders[$row['id']] = $row['path'];
$offset = 0;
$step = 500;
$continue = true;
$count = 0;
foreach( XMLImportDB::arrayQuery( "SELECT count(*) AS c FROM content" ) as $row )
$count = $row['c'];
$cli->output( 'Updating taxonomies for content_ids - '.$count.' objects found' );
do
{
$sql = sprintf( "SELECT c.id,
GROUP_CONCAT(a.language SEPARATOR '|') as lang,
c.publisher_folder_id
FROM content c,
article a
WHERE c.id = a.content_id
AND c.id BETWEEN %d AND %d
GROUP BY c.id
", $offset, ( $offset + $step - 1 ) );
self::$publishersAndLanguages = array();
foreach( XMLImportDB::arrayQuery($sql) as $row )
self::$publishersAndLanguages[$row['id']] = array( 'lang' => explode('|', $row['lang']), 'publisher' => $row['publisher_folder_id'] );
$sql = sprintf( "SELECT ct.content_id,
ct.taxonomy_id
FROM content_has_taxonomy ct
WHERE ct.content_id BETWEEN %d AND %d
ORDER BY ct.content_id
", $offset, ( $offset + $step - 1 ) );
$cli->output( str_pad( number_format( ($offset + $step ) / $count * 100, 2 ), 6, ' ', STR_PAD_LEFT )."% ".str_pad( $offset, 10, ' ', STR_PAD_LEFT).' > '.str_pad( $offset + $step - 1, 10, ' ', STR_PAD_LEFT ) );
self::$dbEz->begin();
$rows = XMLImportDB::arrayQuery( $sql );
$continue = count($rows);
$objectId = null;
$taxonomies = array();
foreach( $rows as $row )
{
if( !is_null($objectId) && $objectId != $row['content_id'] )
{
self::updateObject( $objectId, $taxonomies );
$objectId = $row['content_id'];
$taxonomies = array( $row['taxonomy_id'] );
}
else
{
$objectId = $row['content_id'];
$taxonomies[] = $row['taxonomy_id'];
}
}
if( $objectId )
self::updateObject( $objectId, $taxonomies );
$offset += $step;
self::$dbEz->commit();
}
while($continue);
}
if ( $apply )
$cli->notice('Update...');
foreach ( $mappingOldIDWithKey as $id => $key )
{
if ( !isset($mappingKeyWithNewID[$key]) )
{
$cli->warning(sprintf('No corresponding between old (%s) and new ID for the key %s', $id, $key));
continue;
}
$newId = $mappingKeyWithNewID[$key];
$sql = sprintf('UPDATE content_has_taxonomy SET new_taxonomy_id = %s WHERE taxonomy_id = %s;', $newId, $id);
if ( $apply )
{
XMLImportDB::query($sql);
}
}
if ( $apply && $tmpColumnExist )
{
$qCli = new QuestionInteractiveCli();
$switchTable = $qCli->askQuestion('Switch actual column by the temporary column', array(
'Y' => 'New values replace the old and old value are maintained in a new column',
'N' => 'Do nothing : old values are maintained and the new value remain in the temporary column'
));
if ( $switchTable == 'Y' )
{
