Beispiel #1
0
 /**
  * Get an existing run
  *
  * @param $intRun
  * @return \Model|null
  */
 public static function getRun($intRun)
 {
     // only runs that are not closed
     $objRun = self::findOneBy(array('tl_convertx_run.id=?', 'tl_convertx_run.closed!=1'), $intRun);
     $objRun->steps = Helper::arrayOnly($objRun->steps);
     $objRun->targets = Helper::arrayOnly($objRun->targets);
     $objRun->sources = Helper::arrayOnly($objRun->sources);
     $objRun->filled = Helper::arrayOnly($objRun->filled);
     $objRun->cleared = Helper::arrayOnly($objRun->cleared);
     return $objRun;
 }
Beispiel #2
0
 /**
  * Find the internal and external target tables (tmp versions)
  *
  * @param $intPid
  * @return array
  */
 public static function findTargetTables($intPid)
 {
     $objJob = JobModel::findByPk($intPid);
     $arrTargets = Helper::arrayOnly($objJob->targetTables);
     $objSteps = StepModel::findBy(array('tl_convertx_step.pid=?', 'tl_convertx_step.published=?', 'tl_convertx_step.action=?'), array($intPid, 1, 'converter'));
     if (!$objSteps) {
         return $arrTargets;
     }
     while ($objSteps->next()) {
         $objConverter = ConverterModel::findByPk($objSteps->converter);
         switch ($objConverter->targetType) {
             case 'InternalTable':
                 // table names for internal data
                 $arrTargets[] = 'cvx_' . $objConverter->targetTable;
                 break;
             default:
                 // tmp tables for external data get a name based on the converter id
                 $arrTargets[] = 'cvx_' . $objConverter->id;
                 break;
         }
     }
     return array_unique($arrTargets);
 }
Beispiel #3
0
 /**
  * Get registered hooks that affect fields
  *
  * @param $arrRow
  * @return array
  */
 public function getHooks($arrRow)
 {
     return Helper::getHooks('field');
 }
Beispiel #4
0
 /**
  * Get registered hooks
  *
  * @param $arrRow
  * @return array
  */
 public function getHooks($arrRow)
 {
     return Helper::getHooks('step');
 }
Beispiel #5
0
 /**
  * Prepare the tmp table for updates, use the deletion rules for it
  *
  * @param $objRun
  * @return bool
  */
 public static function clearTable($objRun)
 {
     $Database = Database::getInstance();
     // perform the deletion rules defined in the converter for every internal target table
     foreach ($objRun->targets as $strTarget) {
         if (!in_array($strTarget, $objRun->cleared)) {
             $objConverter = false;
             foreach ($objRun->steps as $intStep) {
                 $objStep = StepModel::findByPk($intStep);
                 if ($objStep->action == 'converter' && !$objConverter) {
                     $objConverter = ConverterModel::findOneBy(array('id=?', 'targetTable=?'), array($objStep->converter, str_replace('cvx_', '', $strTarget)));
                 }
             }
             // there are tables not bound on a converter
             if (!$objConverter) {
                 return $strTarget;
             } else {
                 $objConverter->deleteRules = Helper::arrayOnly($objConverter->deleteRules);
                 $objConverter->fieldsTarget = Helper::arrayOnly($objConverter->fieldsTarget);
                 $objConverter->fieldsSource = Helper::arrayOnly($objConverter->fieldsSource);
                 // for updates we need all existent keys of the source and the target table
                 if ($objConverter->allowUpdate || $objConverter->deleteOnStart != '' && $objConverter->deleteOnStart != 'all') {
                     // source
                     $strSource = $objConverter->sourceType == 'InternalTable' ? $objConverter->sourceTable : 'cvx_' . $objConverter->id . '_source';
                     $objKeys = $Database->prepare("SELECT " . $objConverter->keySource . " FROM " . $strSource)->execute();
                     $arrSourceKeys = $objKeys->fetchEach($objConverter->keySource);
                     if ($objConverter->sourceHasFieldnames) {
                         // seems to be an artefact
                         //                            array_shift($arrSourceKeys);
                     }
                     // target
                     $objKeys = $Database->prepare("SELECT " . $objConverter->keyTarget . " FROM " . $strTarget)->execute();
                     $arrTargetKeys = $objKeys->fetchEach($objConverter->keyTarget);
                     $objConverter->targetKeys = serialize($arrTargetKeys);
                 }
                 $objConverter->save();
                 $blnOk = true;
                 // delete all data in target table
                 if ($objConverter->deleteOnStart == 'all') {
                     if (!($objDeletion = $Database->prepare("TRUNCATE TABLE " . $strTarget)->execute())) {
                         $blnOk = false;
                     }
                 } elseif ($objConverter->deleteOnStart == 'missing') {
                     // delete data missing in target, but existing in the source table
                     if (!($objDeletion = $Database->prepare("DELETE FROM " . $strTarget . " WHERE " . $objConverter->keyTarget . " IN ('" . implode("','", array_diff($arrTargetKeys, $arrSourceKeys)) . "')")->execute())) {
                         $blnOk = false;
                     }
                 } elseif ($objConverter->deleteOnStart == 'existent') {
                     // delete data already existing in target table
                     if (!($objDeletion = $Database->prepare("DELETE FROM " . $strTarget . " WHERE " . $objConverter->keyTarget . " IN ('" . implode("','", array_diff($arrTargetKeys, array_diff($arrTargetKeys, $arrSourceKeys))) . "')")->execute())) {
                         $blnOk = false;
                     }
                 }
                 // delete depending on rules
                 if ($objConverter->deleteOnRules) {
                     $strWhere = '';
                     foreach ($objConverter->deleteRules as $k => $v) {
                         $strWhere .= ($k > 0 ? ' ' . $v['type'] . ' ' : '') . $v['field'] . ' ';
                         switch ($v['operator']) {
                             case 'gteq':
                                 $strWhere .= '>= \'' . addslashes($v['value']) . '\'';
                                 break;
                             case 'gt':
                                 $strWhere .= '> \'' . addslashes($v['value']) . '\'';
                                 break;
                             case 'lt':
                                 $strWhere .= '< \'' . addslashes($v['value']) . '\'';
                                 break;
                             case 'lteq':
                                 $strWhere .= '<= \'' . addslashes($v['value']) . '\'';
                                 break;
                             case 'begins':
                                 $strWhere .= 'LIKE \'' . addslashes($v['value']) . '%\'';
                                 break;
                             case 'ends':
                                 $strWhere .= 'LIKE \'%' . addslashes($v['value']) . '\'';
                                 break;
                             default:
                                 $strWhere .= '= \'' . addslashes($v['value']) . '\'';
                                 break;
                         }
                     }
                     if (!($objDeletion = $Database->prepare("DELETE FROM " . $strTarget . " WHERE " . $strWhere)->execute())) {
                         $blnOk = false;
                     }
                 }
                 if ($blnOk) {
                     Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['temporaryTargetCleared'], $strTarget), 'entry');
                     return $strTarget;
                 } else {
                     Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['temporaryTargetNotCleared'], $strTarget), 'entry', 'error');
                     return false;
                 }
             }
             // fill exactly 1 of the temp source tables at a time
             continue;
         }
     }
 }
 /**
  * Do the raw data import
  *
  * @param $intConverter
  * @param $objRun
  * @return bool|mixed
  */
 public static function rawImport($intConverter, $objRun)
 {
     // tmp table mit csv befüllen
     $objConverter = ConverterModel::findByPk($intConverter);
     $arrFields = unserialize($objConverter->fieldsSource);
     foreach ($arrFields as $k => $v) {
         $arrFieldNames[] = $v['name'];
     }
     // the csv file
     $objCsv = FilesModel::findByUuid($objConverter->sourceFile);
     //print_r($objCsv);die();
     // set or determine char set
     $strCharset = $objConverter->sourceCharSet ? $objConverter->sourceCharSet : false;
     // set or determine line feed
     $strLinefeed = $objConverter->sourceLineFeed ? $GLOBALS['convertx']['csv_linefeeds'][$objConverter->sourceLineFeed] : false;
     // determine charset and/or line feed
     if (!$strCharset || !$strLinefeed) {
         $objFile = new File($objCsv->path);
         $strCompleteFile = $objFile->getContent();
         $strFirstLine = fgets($objFile->handle, 40960);
         $strCharset = $strCharset ? $strCharset : Helper::getEncoding($strCompleteFile);
         $strLinefeed = $strLinefeed ? $strLinefeed : Helper::getLineFeed($strFirstLine);
     }
     // set or determine enclosure
     $strEnclosure = $objConverter->sourceEnclosure ? '\\' . $GLOBALS['convertx']['csv_enclosures'][$objConverter->sourceEnclosure] : '\\"';
     // delete all data in the source table (maybe there after an error before)
     Database::getInstance()->prepare("TRUNCATE cvx_" . $intConverter . "_source")->execute();
     if (!$objConverter->sourceUseDataInfile) {
         // handle csv file line by line
         $objFile = new File($objCsv->path);
         $arrCompleteFile = $objFile->getContentAsArray();
         foreach ($arrCompleteFile as $k => $v) {
             if ($k > 0 || !$objConverter->sourceHasFieldnames) {
                 $arrLine = String::splitCsv($v, $GLOBALS['convertx']['csv_delimiters'][$objConverter->sourceFieldDelimiter]);
                 $arrSet = array();
                 foreach ($arrFieldNames as $k2 => $v2) {
                     $arrSet[$v2] = $arrLine[$k2];
                 }
                 Database::getInstance()->prepare("insert into cvx_" . $intConverter . "_source %s")->set($arrSet)->execute();
             }
         }
     } else {
         // load it directly into the database
         $strSql = sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE `%s` CHARACTER SET %s FIELDS TERMINATED BY '%s' OPTIONALLY ENCLOSED BY '%s' LINES TERMINATED BY '%s' IGNORE %s LINES (`%s`)", addslashes(TL_ROOT . '/' . $objCsv->path), 'cvx_' . $intConverter . '_source', $strCharset, $GLOBALS['convertx']['csv_delimiters'][$objConverter->sourceFieldDelimiter], $strEnclosure, $strLinefeed, $objConverter->sourceHasFieldnames ? '1' : '0', implode('`, `', $arrFieldNames));
         Database::getInstance()->prepare($strSql)->execute();
     }
     // try to eliminate lines that are completely empty
     Database::getInstance()->prepare("DELETE FROM cvx_" . $intConverter . "_source WHERE `" . implode("`='' AND `", $arrFieldNames) . "`=''")->execute();
     return true;
 }
Beispiel #7
0
 /**
  * Perform the step
  *
  * @param $objRun
  * @param $objStep
  * @return bool|mixed|object
  */
 public static function doStep($objRun, $objStep)
 {
     $objConverter = ConverterModel::findByPk($objStep->converter);
     $objConverter->fieldsTarget = Helper::arrayOnly($objConverter->fieldsTarget);
     $objConverter->fieldsSource = Helper::arrayOnly($objConverter->fieldsSource);
     $objConverter->targetKeys = Helper::arrayOnly($objConverter->targetKeys);
     // -----------------------------------------
     // walk thru source data
     // source
     if ($objConverter->sourceType == 'InternalTable') {
         $strSource = $objConverter->useTempSource ? 'cvx_' . $objConverter->sourceTable : $objConverter->sourceTable;
     } else {
         $strSource = 'cvx_' . $objConverter->id . '_source';
     }
     $objSource = Database::getInstance()->prepare("SELECT * FROM " . $strSource)->execute();
     $strKeySource = $objConverter->keySource;
     $intBegin = time();
     $intLine = 0;
     while ($objSource->next()) {
         $intLine++;
         // skip already processed lines
         if ($objRun->line > 0 && $intLine <= $objRun->line) {
             continue;
         }
         // perform an update if possible (allowed, key exists, target table not truncated)
         if ($objConverter->allowUpdate && in_array($objSource->{$strKeySource}, $objConverter->targetKeys) && $objConverter->deleteOnStart != 'all') {
             if ($strUpdated = self::insertOrUpdate('update', $objRun, $objConverter, $objSource)) {
                 $arrUpdated[] = $strUpdated;
             } else {
                 $hasErrors = true;
                 // abort step if 1 line fails
                 if ($objConverter->abortOnError) {
                     break;
                 }
             }
         } elseif ($objConverter->allowInsert) {
             //            elseif ($objConverter->allowInsert && (!$objConverter->allowUpdate || ($objConverter->allowUpdate && !in_array($objSource->$strKeySource, $objConverter->targetKeys)))) {
             if ($strInserted = self::insertOrUpdate('insert', $objRun, $objConverter, $objSource)) {
                 if ($strInserted != 'skip') {
                     $arrInserted[] = $strInserted;
                 }
             } else {
                 $hasErrors = true;
                 // abort step if 1 line fails
                 if ($objConverter->abortOnError) {
                     break;
                 }
             }
         }
         // check for performance of this step within the max execution time
         $intNow = time();
         if ($intNow > $intBegin + $GLOBALS['convertx']['maxExecutionTime']) {
             $reload = true;
             break;
         }
     }
     // -----------------------------------------
     // an error may not lead to a step abortion
     if ($hasErrors && $objConverter->abortOnError) {
         if ($objStep->abortOnError) {
             $objReturn = (object) null;
             $objReturn->title = $GLOBALS['TL_LANG']['tl_convertx_job']['abortion'];
             $objReturn->error = sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['abortionStep'], $objStep->title);
             return $objReturn;
         }
     }
     // -----------------------------------------
     // note the amount of manipulated data sets
     if (count($arrInserted) > 0) {
         Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['setsInserted'], count($arrInserted)), 'entry', 'ok', print_r($arrInserted, true));
     }
     if (count($arrUpdated) > 0) {
         Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['setsUpdated'], count($arrUpdated)), 'entry', 'ok', print_r($arrUpdated, true));
     }
     if ($reload) {
         $objReturn = (object) null;
         $objReturn->title = $GLOBALS['TL_LANG']['tl_convertx_job']['splitting'];
         $objReturn->line = $intLine;
         return $objReturn;
     }
     return true;
 }
Beispiel #8
0
 /**
  * Get field names for the source key definition
  *
  * @param $obj
  * @return array
  */
 public function getSourceFields($obj)
 {
     if (!$obj->activeRecord->fieldsSource) {
         return array();
     }
     $arrFields = Helper::arrayOnly($obj->activeRecord->fieldsSource);
     foreach ($arrFields as $arrField) {
         $arrReturn[] = $arrField['name'];
     }
     return $arrReturn;
 }