Example #1
0
 /**
  * List steps
  *
  * @param $arrRow
  * @return string
  */
 public function listSteps($arrRow)
 {
     if ($arrRow['converter']) {
         $objConverter = ConverterModel::findByPk($arrRow['converter']);
     }
     $key = $arrRow['published'] ? 'published' : 'unpublished';
     $return = '<div class="cte_type ' . $key . '"><strong>' . $arrRow['title'] . '</strong></div>';
     $return .= $arrRow['action'] == 'converter' ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_step']['references'][$arrRow['action']] . ': ' . $objConverter->title . ' (' . $arrRow['converter'] . ')' . '</div>' : '';
     $return .= $arrRow['action'] == 'sql' ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_step']['references'][$arrRow['action']] . '</div>' : '';
     $return .= $arrRow['action'] == 'hook' ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_step']['references'][$arrRow['action']] . ': ' . $arrRow['hook'] . '</div>' : '';
     $return .= $arrRow['abortOnError'] ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_step']['abortOnError'][0] . '</div>' : '';
     $return .= "\n";
     return $return;
 }
Example #2
0
 /**
  * Fill the tmp table in the way specified in the dada converter
  *
  * @param $objRun
  * @return bool|mixed
  */
 public static function fillTable($objRun)
 {
     foreach ($objRun->sources as $strSource) {
         if (!in_array($strSource, $objRun->filled)) {
             $intConverter = str_replace('_source', '', str_replace('cvx_', '', $strSource));
             $objConverter = ConverterModel::findByPk($intConverter);
             // get the glass of the converter
             $strClass = ($GLOBALS['convertx']['classpath'][$objConverter->sourceType] ? $GLOBALS['convertx']['classpath'][$objConverter->sourceType] : 'Delahaye\\ConvertX\\Container') . '\\' . $objConverter->sourceType;
             // fill temp table
             if ($strClass::rawImport($intConverter, $objRun)) {
                 Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['temporarySourceFilled'], $intConverter), 'entry');
                 return $intConverter;
             } else {
                 Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['temporarySourceNotFilled'], $intConverter), 'entry', 'error');
                 return false;
             }
             // fill exactly 1 of the temp source tables at a time
             continue;
         }
     }
 }
Example #3
0
 /**
  * build a config for the target fields of the converter
  *
  * @param $intConverter
  */
 public static function createTargetFields($intConverter)
 {
     $objConverter = Converter::findByPk($intConverter);
     $objConverterfields = Converterfield::findBy('pid', $intConverter);
     // -----------------------------------------
     // check if we already have some field definitions, only add new
     if ($objConverterfields) {
         while ($objConverterfields->next()) {
             $arrFieldsSource = $arrFieldsSource ? $arrFieldsSource : unserialize($objConverter->fieldsSource);
             $arrFieldsTarget = $arrFieldsTarget ? $arrFieldsTarget : unserialize($objConverter->fieldsTarget);
             $BlnAllowInsert = $BlnAllowInsert ? $BlnAllowInsert : $objConverter->allowInsert;
             $BlnAllowUpdate = $BlnAllowUpdate ? $BlnAllowUpdate : $objConverter->allowUpdate;
             foreach ($arrFieldsTarget as $k => $v) {
                 if ($v['name'] == $objConverterfields->fieldname) {
                     unset($arrFieldsTarget[$k]);
                     $sorting[] = $objConverterfields->sorting;
                 }
             }
         }
         rsort($sorting);
         $sorting = $sorting[0];
     } else {
         $arrFieldsSource = unserialize($objConverter->fieldsSource);
         $arrFieldsTarget = unserialize($objConverter->fieldsTarget);
         $BlnAllowInsert = $objConverter->allowInsert;
         $BlnAllowUpdate = $objConverter->allowUpdate;
     }
     // source fields, collect fieldnames
     foreach ($arrFieldsSource as $fieldSource) {
         $arrFieldnamesSource[] = $fieldSource['name'];
     }
     // pre-define converterdefinitions for existing target fields, only 1:1
     foreach ($arrFieldsTarget as $fieldTarget) {
         $sorting = $sorting + 32;
         $arrNew = array('pid' => $intConverter, 'sorting' => $sorting, 'tstamp' => time(), 'published' => '', 'fieldname' => $fieldTarget['name'], 'allowInsert' => $BlnAllowInsert ? $BlnAllowInsert : '', 'typeInsert' => 'Insertfromfield', 'fieldInsert' => in_array($fieldTarget['name'], $arrFieldnamesSource) ? $fieldTarget['name'] : '', 'modeInsert' => 'Addnew', 'allowUpdate' => $BlnAllowUpdate ? $BlnAllowUpdate : '', 'typeUpdate' => 'Updatefromfield', 'fieldUpdate' => in_array($fieldTarget['name'], $arrFieldnamesSource) ? $fieldTarget['name'] : '', 'modeUpdate' => 'Replace');
         $objNew = new Converterfield();
         $objNew->setRow($arrNew);
         $objNew->save();
     }
 }
Example #4
0
 /**
  * List the elements
  *
  * @param $arrRow
  * @return string
  */
 public function listElements($arrRow)
 {
     $objConverter = ConverterModel::findByPk($arrRow['pid']);
     $key = $arrRow['published'] ? 'published' : 'unpublished';
     $return = '<div class="cte_type ' . $key . '"><strong>' . $arrRow['fieldname'] . '</strong> (' . $arrRow['sorting'] . ')</div>';
     $return .= $objConverter->allowInsert ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['new'] . ': ' . ($arrRow['allowInsert'] ? $arrRow['typeInsert'] == 'Insertfromfield' ? $arrRow['fieldInsert'] ? $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['field'] . ' "' . $arrRow['fieldInsert'] . '", ' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['modeInsert']] : (!$arrRow['fieldInsert'] && !$arrRow['modeInsert'] ? $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['tmpField'] : '-') : $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['typeInsert']] : '-') . '</div>' : '';
     $return .= $objConverter->allowUpdate ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['update'] . ': ' . ($arrRow['allowUpdate'] ? $arrRow['typeUpdate'] == 'Updatefromfield' ? $arrRow['fieldUpdate'] ? $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['field'] . ' "' . $arrRow['fieldUpdate'] . '", ' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['modeUpdate']] : (!$arrRow['fieldInsert'] && !$arrRow['fieldUpdate'] ? $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['tmpField'] . ', ' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['modeUpdate']] : '-') : $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['typeUpdate']] : '-') . '</div>' : '';
     $return .= "\n";
     return $return;
 }
Example #5
0
 /**
  * Find the internal and external source tables (tmp versions)
  *
  * @param $intPid
  * @return array
  */
 public static function findTmpSourceTables($intPid)
 {
     $objSteps = StepModel::findBy(array('tl_convertx_step.pid=?', 'tl_convertx_step.published=?', 'tl_convertx_step.action=?'), array($intPid, 1, 'converter'));
     if (!$objSteps) {
         return array();
     }
     while ($objSteps->next()) {
         $objConverter = ConverterModel::findByPk($objSteps->converter);
         switch ($objConverter->sourceType) {
             case 'InternalTable':
                 // for internal sources we don't need tmp tables
                 break;
             default:
                 // tmp tables for external data get a name based on the converter id
                 $arrSources[] = 'cvx_' . $objConverter->id . '_source';
                 break;
         }
     }
     return (array) $arrSources;
 }
 /**
  * Do the raw data import
  *
  * @param $intConverter
  * @param $objRun
  * @return bool|mixed
  */
 public static function rawImport($intConverter, $objRun)
 {
     // tmp table mit csv befüllen
     $objConverter = ConverterModel::findByPk($intConverter);
     $arrFields = unserialize($objConverter->fieldsSource);
     foreach ($arrFields as $k => $v) {
         $arrFieldNames[] = $v['name'];
     }
     // the csv file
     $objCsv = FilesModel::findByUuid($objConverter->sourceFile);
     //print_r($objCsv);die();
     // set or determine char set
     $strCharset = $objConverter->sourceCharSet ? $objConverter->sourceCharSet : false;
     // set or determine line feed
     $strLinefeed = $objConverter->sourceLineFeed ? $GLOBALS['convertx']['csv_linefeeds'][$objConverter->sourceLineFeed] : false;
     // determine charset and/or line feed
     if (!$strCharset || !$strLinefeed) {
         $objFile = new File($objCsv->path);
         $strCompleteFile = $objFile->getContent();
         $strFirstLine = fgets($objFile->handle, 40960);
         $strCharset = $strCharset ? $strCharset : Helper::getEncoding($strCompleteFile);
         $strLinefeed = $strLinefeed ? $strLinefeed : Helper::getLineFeed($strFirstLine);
     }
     // set or determine enclosure
     $strEnclosure = $objConverter->sourceEnclosure ? '\\' . $GLOBALS['convertx']['csv_enclosures'][$objConverter->sourceEnclosure] : '\\"';
     // delete all data in the source table (maybe there after an error before)
     Database::getInstance()->prepare("TRUNCATE cvx_" . $intConverter . "_source")->execute();
     if (!$objConverter->sourceUseDataInfile) {
         // handle csv file line by line
         $objFile = new File($objCsv->path);
         $arrCompleteFile = $objFile->getContentAsArray();
         foreach ($arrCompleteFile as $k => $v) {
             if ($k > 0 || !$objConverter->sourceHasFieldnames) {
                 $arrLine = String::splitCsv($v, $GLOBALS['convertx']['csv_delimiters'][$objConverter->sourceFieldDelimiter]);
                 $arrSet = array();
                 foreach ($arrFieldNames as $k2 => $v2) {
                     $arrSet[$v2] = $arrLine[$k2];
                 }
                 Database::getInstance()->prepare("insert into cvx_" . $intConverter . "_source %s")->set($arrSet)->execute();
             }
         }
     } else {
         // load it directly into the database
         $strSql = sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE `%s` CHARACTER SET %s FIELDS TERMINATED BY '%s' OPTIONALLY ENCLOSED BY '%s' LINES TERMINATED BY '%s' IGNORE %s LINES (`%s`)", addslashes(TL_ROOT . '/' . $objCsv->path), 'cvx_' . $intConverter . '_source', $strCharset, $GLOBALS['convertx']['csv_delimiters'][$objConverter->sourceFieldDelimiter], $strEnclosure, $strLinefeed, $objConverter->sourceHasFieldnames ? '1' : '0', implode('`, `', $arrFieldNames));
         Database::getInstance()->prepare($strSql)->execute();
     }
     // try to eliminate lines that are completely empty
     Database::getInstance()->prepare("DELETE FROM cvx_" . $intConverter . "_source WHERE `" . implode("`='' AND `", $arrFieldNames) . "`=''")->execute();
     return true;
 }
Example #7
0
 /**
  * Perform the step
  *
  * @param $objRun
  * @param $objStep
  * @return bool|mixed|object
  */
 public static function doStep($objRun, $objStep)
 {
     $objConverter = ConverterModel::findByPk($objStep->converter);
     $objConverter->fieldsTarget = Helper::arrayOnly($objConverter->fieldsTarget);
     $objConverter->fieldsSource = Helper::arrayOnly($objConverter->fieldsSource);
     $objConverter->targetKeys = Helper::arrayOnly($objConverter->targetKeys);
     // -----------------------------------------
     // walk thru source data
     // source
     if ($objConverter->sourceType == 'InternalTable') {
         $strSource = $objConverter->useTempSource ? 'cvx_' . $objConverter->sourceTable : $objConverter->sourceTable;
     } else {
         $strSource = 'cvx_' . $objConverter->id . '_source';
     }
     $objSource = Database::getInstance()->prepare("SELECT * FROM " . $strSource)->execute();
     $strKeySource = $objConverter->keySource;
     $intBegin = time();
     $intLine = 0;
     while ($objSource->next()) {
         $intLine++;
         // skip already processed lines
         if ($objRun->line > 0 && $intLine <= $objRun->line) {
             continue;
         }
         // perform an update if possible (allowed, key exists, target table not truncated)
         if ($objConverter->allowUpdate && in_array($objSource->{$strKeySource}, $objConverter->targetKeys) && $objConverter->deleteOnStart != 'all') {
             if ($strUpdated = self::insertOrUpdate('update', $objRun, $objConverter, $objSource)) {
                 $arrUpdated[] = $strUpdated;
             } else {
                 $hasErrors = true;
                 // abort step if 1 line fails
                 if ($objConverter->abortOnError) {
                     break;
                 }
             }
         } elseif ($objConverter->allowInsert) {
             //            elseif ($objConverter->allowInsert && (!$objConverter->allowUpdate || ($objConverter->allowUpdate && !in_array($objSource->$strKeySource, $objConverter->targetKeys)))) {
             if ($strInserted = self::insertOrUpdate('insert', $objRun, $objConverter, $objSource)) {
                 if ($strInserted != 'skip') {
                     $arrInserted[] = $strInserted;
                 }
             } else {
                 $hasErrors = true;
                 // abort step if 1 line fails
                 if ($objConverter->abortOnError) {
                     break;
                 }
             }
         }
         // check for performance of this step within the max execution time
         $intNow = time();
         if ($intNow > $intBegin + $GLOBALS['convertx']['maxExecutionTime']) {
             $reload = true;
             break;
         }
     }
     // -----------------------------------------
     // an error may not lead to a step abortion
     if ($hasErrors && $objConverter->abortOnError) {
         if ($objStep->abortOnError) {
             $objReturn = (object) null;
             $objReturn->title = $GLOBALS['TL_LANG']['tl_convertx_job']['abortion'];
             $objReturn->error = sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['abortionStep'], $objStep->title);
             return $objReturn;
         }
     }
     // -----------------------------------------
     // note the amount of manipulated data sets
     if (count($arrInserted) > 0) {
         Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['setsInserted'], count($arrInserted)), 'entry', 'ok', print_r($arrInserted, true));
     }
     if (count($arrUpdated) > 0) {
         Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['setsUpdated'], count($arrUpdated)), 'entry', 'ok', print_r($arrUpdated, true));
     }
     if ($reload) {
         $objReturn = (object) null;
         $objReturn->title = $GLOBALS['TL_LANG']['tl_convertx_job']['splitting'];
         $objReturn->line = $intLine;
         return $objReturn;
     }
     return true;
 }
Example #8
0
 /**
  * Save field lists for target and source
  *
  * @param DataContainer $dc
  */
 public function setFieldLists(DataContainer $dc)
 {
     $objData = ConverterModel::findByPk(Input::get('id'));
     $classTarget = $dc->activeRecord->targetType ? ($GLOBALS['convertx']['classpath'][$dc->activeRecord->targetType] ? $GLOBALS['convertx']['classpath'][$dc->activeRecord->targetType] : 'Delahaye\\ConvertX\\Container') . '\\' . $dc->activeRecord->targetType : false;
     $classSource = $dc->activeRecord->sourceType ? ($GLOBALS['convertx']['classpath'][$dc->activeRecord->sourceType] ? $GLOBALS['convertx']['classpath'][$dc->activeRecord->sourceType] : 'Delahaye\\ConvertX\\Container') . '\\' . $dc->activeRecord->sourceType : false;
     // check if target fields have to be cleared
     if ($classTarget) {
         if ($classTarget::checkTargetClear($dc, $objData)) {
             ConverterfieldModel::clearTargetFields(Input::get('id'));
         }
     }
     // use actual data
     foreach ($GLOBALS['TL_DCA']['tl_convertx_converter']['fields'] as $k => $v) {
         $objData->{$k} = $dc->activeRecord->{$k};
     }
     // get field lists for source and/or target
     if ($classTarget) {
         $objData->fieldsTarget = $dc->activeRecord->targetType ? serialize($classTarget::getFieldList($dc->activeRecord, 'target')) : false;
     }
     if ($classSource) {
         $objData->fieldsSource = $dc->activeRecord->sourceType ? serialize($classSource::getFieldList($dc->activeRecord, 'source')) : false;
     }
     // save the model
     $objData->save();
     return;
 }