/**
* List steps
*
* @param $arrRow
* @return string
*/
public function listSteps($arrRow)
{
if ($arrRow['converter']) {
$objConverter = ConverterModel::findByPk($arrRow['converter']);
}
$key = $arrRow['published'] ? 'published' : 'unpublished';
$return = '<div class="cte_type ' . $key . '"><strong>' . $arrRow['title'] . '</strong></div>';
$return .= $arrRow['action'] == 'converter' ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_step']['references'][$arrRow['action']] . ': ' . $objConverter->title . ' (' . $arrRow['converter'] . ')' . '</div>' : '';
$return .= $arrRow['action'] == 'sql' ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_step']['references'][$arrRow['action']] . '</div>' : '';
$return .= $arrRow['action'] == 'hook' ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_step']['references'][$arrRow['action']] . ': ' . $arrRow['hook'] . '</div>' : '';
$return .= $arrRow['abortOnError'] ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_step']['abortOnError'][0] . '</div>' : '';
$return .= "\n";
return $return;
}
/**
* Fill the tmp table in the way specified in the dada converter
*
* @param $objRun
* @return bool|mixed
*/
public static function fillTable($objRun)
{
foreach ($objRun->sources as $strSource) {
if (!in_array($strSource, $objRun->filled)) {
$intConverter = str_replace('_source', '', str_replace('cvx_', '', $strSource));
$objConverter = ConverterModel::findByPk($intConverter);
// get the glass of the converter
$strClass = ($GLOBALS['convertx']['classpath'][$objConverter->sourceType] ? $GLOBALS['convertx']['classpath'][$objConverter->sourceType] : 'Delahaye\\ConvertX\\Container') . '\\' . $objConverter->sourceType;
// fill temp table
if ($strClass::rawImport($intConverter, $objRun)) {
Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['temporarySourceFilled'], $intConverter), 'entry');
return $intConverter;
} else {
Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['temporarySourceNotFilled'], $intConverter), 'entry', 'error');
return false;
}
// fill exactly 1 of the temp source tables at a time
continue;
}
}
}
/**
* build a config for the target fields of the converter
*
* @param $intConverter
*/
public static function createTargetFields($intConverter)
{
$objConverter = Converter::findByPk($intConverter);
$objConverterfields = Converterfield::findBy('pid', $intConverter);
// -----------------------------------------
// check if we already have some field definitions, only add new
if ($objConverterfields) {
while ($objConverterfields->next()) {
$arrFieldsSource = $arrFieldsSource ? $arrFieldsSource : unserialize($objConverter->fieldsSource);
$arrFieldsTarget = $arrFieldsTarget ? $arrFieldsTarget : unserialize($objConverter->fieldsTarget);
$BlnAllowInsert = $BlnAllowInsert ? $BlnAllowInsert : $objConverter->allowInsert;
$BlnAllowUpdate = $BlnAllowUpdate ? $BlnAllowUpdate : $objConverter->allowUpdate;
foreach ($arrFieldsTarget as $k => $v) {
if ($v['name'] == $objConverterfields->fieldname) {
unset($arrFieldsTarget[$k]);
$sorting[] = $objConverterfields->sorting;
}
}
}
rsort($sorting);
$sorting = $sorting[0];
} else {
$arrFieldsSource = unserialize($objConverter->fieldsSource);
$arrFieldsTarget = unserialize($objConverter->fieldsTarget);
$BlnAllowInsert = $objConverter->allowInsert;
$BlnAllowUpdate = $objConverter->allowUpdate;
}
// source fields, collect fieldnames
foreach ($arrFieldsSource as $fieldSource) {
$arrFieldnamesSource[] = $fieldSource['name'];
}
// pre-define converterdefinitions for existing target fields, only 1:1
foreach ($arrFieldsTarget as $fieldTarget) {
$sorting = $sorting + 32;
$arrNew = array('pid' => $intConverter, 'sorting' => $sorting, 'tstamp' => time(), 'published' => '', 'fieldname' => $fieldTarget['name'], 'allowInsert' => $BlnAllowInsert ? $BlnAllowInsert : '', 'typeInsert' => 'Insertfromfield', 'fieldInsert' => in_array($fieldTarget['name'], $arrFieldnamesSource) ? $fieldTarget['name'] : '', 'modeInsert' => 'Addnew', 'allowUpdate' => $BlnAllowUpdate ? $BlnAllowUpdate : '', 'typeUpdate' => 'Updatefromfield', 'fieldUpdate' => in_array($fieldTarget['name'], $arrFieldnamesSource) ? $fieldTarget['name'] : '', 'modeUpdate' => 'Replace');
$objNew = new Converterfield();
$objNew->setRow($arrNew);
$objNew->save();
}
}
/**
* List the elements
*
* @param $arrRow
* @return string
*/
public function listElements($arrRow)
{
$objConverter = ConverterModel::findByPk($arrRow['pid']);
$key = $arrRow['published'] ? 'published' : 'unpublished';
$return = '<div class="cte_type ' . $key . '"><strong>' . $arrRow['fieldname'] . '</strong> (' . $arrRow['sorting'] . ')</div>';
$return .= $objConverter->allowInsert ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['new'] . ': ' . ($arrRow['allowInsert'] ? $arrRow['typeInsert'] == 'Insertfromfield' ? $arrRow['fieldInsert'] ? $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['field'] . ' "' . $arrRow['fieldInsert'] . '", ' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['modeInsert']] : (!$arrRow['fieldInsert'] && !$arrRow['modeInsert'] ? $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['tmpField'] : '-') : $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['typeInsert']] : '-') . '</div>' : '';
$return .= $objConverter->allowUpdate ? '<div>' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['update'] . ': ' . ($arrRow['allowUpdate'] ? $arrRow['typeUpdate'] == 'Updatefromfield' ? $arrRow['fieldUpdate'] ? $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['field'] . ' "' . $arrRow['fieldUpdate'] . '", ' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['modeUpdate']] : (!$arrRow['fieldInsert'] && !$arrRow['fieldUpdate'] ? $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references']['tmpField'] . ', ' . $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['modeUpdate']] : '-') : $GLOBALS['TL_LANG']['tl_convertx_converterfield']['references'][$arrRow['typeUpdate']] : '-') . '</div>' : '';
$return .= "\n";
return $return;
}
/**
* Find the internal and external source tables (tmp versions)
*
* @param $intPid
* @return array
*/
public static function findTmpSourceTables($intPid)
{
$objSteps = StepModel::findBy(array('tl_convertx_step.pid=?', 'tl_convertx_step.published=?', 'tl_convertx_step.action=?'), array($intPid, 1, 'converter'));
if (!$objSteps) {
return array();
}
while ($objSteps->next()) {
$objConverter = ConverterModel::findByPk($objSteps->converter);
switch ($objConverter->sourceType) {
case 'InternalTable':
// for internal sources we don't need tmp tables
break;
default:
// tmp tables for external data get a name based on the converter id
$arrSources[] = 'cvx_' . $objConverter->id . '_source';
break;
}
}
return (array) $arrSources;
}
/**
* Do the raw data import
*
* @param $intConverter
* @param $objRun
* @return bool|mixed
*/
public static function rawImport($intConverter, $objRun)
{
// tmp table mit csv befüllen
$objConverter = ConverterModel::findByPk($intConverter);
$arrFields = unserialize($objConverter->fieldsSource);
foreach ($arrFields as $k => $v) {
$arrFieldNames[] = $v['name'];
}
// the csv file
$objCsv = FilesModel::findByUuid($objConverter->sourceFile);
//print_r($objCsv);die();
// set or determine char set
$strCharset = $objConverter->sourceCharSet ? $objConverter->sourceCharSet : false;
// set or determine line feed
$strLinefeed = $objConverter->sourceLineFeed ? $GLOBALS['convertx']['csv_linefeeds'][$objConverter->sourceLineFeed] : false;
// determine charset and/or line feed
if (!$strCharset || !$strLinefeed) {
$objFile = new File($objCsv->path);
$strCompleteFile = $objFile->getContent();
$strFirstLine = fgets($objFile->handle, 40960);
$strCharset = $strCharset ? $strCharset : Helper::getEncoding($strCompleteFile);
$strLinefeed = $strLinefeed ? $strLinefeed : Helper::getLineFeed($strFirstLine);
}
// set or determine enclosure
$strEnclosure = $objConverter->sourceEnclosure ? '\\' . $GLOBALS['convertx']['csv_enclosures'][$objConverter->sourceEnclosure] : '\\"';
// delete all data in the source table (maybe there after an error before)
Database::getInstance()->prepare("TRUNCATE cvx_" . $intConverter . "_source")->execute();
if (!$objConverter->sourceUseDataInfile) {
// handle csv file line by line
$objFile = new File($objCsv->path);
$arrCompleteFile = $objFile->getContentAsArray();
foreach ($arrCompleteFile as $k => $v) {
if ($k > 0 || !$objConverter->sourceHasFieldnames) {
$arrLine = String::splitCsv($v, $GLOBALS['convertx']['csv_delimiters'][$objConverter->sourceFieldDelimiter]);
$arrSet = array();
foreach ($arrFieldNames as $k2 => $v2) {
$arrSet[$v2] = $arrLine[$k2];
}
Database::getInstance()->prepare("insert into cvx_" . $intConverter . "_source %s")->set($arrSet)->execute();
}
}
} else {
// load it directly into the database
$strSql = sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE `%s` CHARACTER SET %s FIELDS TERMINATED BY '%s' OPTIONALLY ENCLOSED BY '%s' LINES TERMINATED BY '%s' IGNORE %s LINES (`%s`)", addslashes(TL_ROOT . '/' . $objCsv->path), 'cvx_' . $intConverter . '_source', $strCharset, $GLOBALS['convertx']['csv_delimiters'][$objConverter->sourceFieldDelimiter], $strEnclosure, $strLinefeed, $objConverter->sourceHasFieldnames ? '1' : '0', implode('`, `', $arrFieldNames));
Database::getInstance()->prepare($strSql)->execute();
}
// try to eliminate lines that are completely empty
Database::getInstance()->prepare("DELETE FROM cvx_" . $intConverter . "_source WHERE `" . implode("`='' AND `", $arrFieldNames) . "`=''")->execute();
return true;
}
/**
* Perform the step
*
* @param $objRun
* @param $objStep
* @return bool|mixed|object
*/
public static function doStep($objRun, $objStep)
{
$objConverter = ConverterModel::findByPk($objStep->converter);
$objConverter->fieldsTarget = Helper::arrayOnly($objConverter->fieldsTarget);
$objConverter->fieldsSource = Helper::arrayOnly($objConverter->fieldsSource);
$objConverter->targetKeys = Helper::arrayOnly($objConverter->targetKeys);
// -----------------------------------------
// walk thru source data
// source
if ($objConverter->sourceType == 'InternalTable') {
$strSource = $objConverter->useTempSource ? 'cvx_' . $objConverter->sourceTable : $objConverter->sourceTable;
} else {
$strSource = 'cvx_' . $objConverter->id . '_source';
}
$objSource = Database::getInstance()->prepare("SELECT * FROM " . $strSource)->execute();
$strKeySource = $objConverter->keySource;
$intBegin = time();
$intLine = 0;
while ($objSource->next()) {
$intLine++;
// skip already processed lines
if ($objRun->line > 0 && $intLine <= $objRun->line) {
continue;
}
// perform an update if possible (allowed, key exists, target table not truncated)
if ($objConverter->allowUpdate && in_array($objSource->{$strKeySource}, $objConverter->targetKeys) && $objConverter->deleteOnStart != 'all') {
if ($strUpdated = self::insertOrUpdate('update', $objRun, $objConverter, $objSource)) {
$arrUpdated[] = $strUpdated;
} else {
$hasErrors = true;
// abort step if 1 line fails
if ($objConverter->abortOnError) {
break;
}
}
} elseif ($objConverter->allowInsert) {
// elseif ($objConverter->allowInsert && (!$objConverter->allowUpdate || ($objConverter->allowUpdate && !in_array($objSource->$strKeySource, $objConverter->targetKeys)))) {
if ($strInserted = self::insertOrUpdate('insert', $objRun, $objConverter, $objSource)) {
if ($strInserted != 'skip') {
$arrInserted[] = $strInserted;
}
} else {
$hasErrors = true;
// abort step if 1 line fails
if ($objConverter->abortOnError) {
break;
}
}
}
// check for performance of this step within the max execution time
$intNow = time();
if ($intNow > $intBegin + $GLOBALS['convertx']['maxExecutionTime']) {
$reload = true;
break;
}
}
// -----------------------------------------
// an error may not lead to a step abortion
if ($hasErrors && $objConverter->abortOnError) {
if ($objStep->abortOnError) {
$objReturn = (object) null;
$objReturn->title = $GLOBALS['TL_LANG']['tl_convertx_job']['abortion'];
$objReturn->error = sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['abortionStep'], $objStep->title);
return $objReturn;
}
}
// -----------------------------------------
// note the amount of manipulated data sets
if (count($arrInserted) > 0) {
Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['setsInserted'], count($arrInserted)), 'entry', 'ok', print_r($arrInserted, true));
}
if (count($arrUpdated) > 0) {
Tracking::log($objRun->id, $objRun->rootRun, sprintf($GLOBALS['TL_LANG']['tl_convertx_job']['setsUpdated'], count($arrUpdated)), 'entry', 'ok', print_r($arrUpdated, true));
}
if ($reload) {
$objReturn = (object) null;
$objReturn->title = $GLOBALS['TL_LANG']['tl_convertx_job']['splitting'];
$objReturn->line = $intLine;
return $objReturn;
}
return true;
}
/**
* Save field lists for target and source
*
* @param DataContainer $dc
*/
public function setFieldLists(DataContainer $dc)
{
$objData = ConverterModel::findByPk(Input::get('id'));
$classTarget = $dc->activeRecord->targetType ? ($GLOBALS['convertx']['classpath'][$dc->activeRecord->targetType] ? $GLOBALS['convertx']['classpath'][$dc->activeRecord->targetType] : 'Delahaye\\ConvertX\\Container') . '\\' . $dc->activeRecord->targetType : false;
$classSource = $dc->activeRecord->sourceType ? ($GLOBALS['convertx']['classpath'][$dc->activeRecord->sourceType] ? $GLOBALS['convertx']['classpath'][$dc->activeRecord->sourceType] : 'Delahaye\\ConvertX\\Container') . '\\' . $dc->activeRecord->sourceType : false;
// check if target fields have to be cleared
if ($classTarget) {
if ($classTarget::checkTargetClear($dc, $objData)) {
ConverterfieldModel::clearTargetFields(Input::get('id'));
}
}
// use actual data
foreach ($GLOBALS['TL_DCA']['tl_convertx_converter']['fields'] as $k => $v) {
$objData->{$k} = $dc->activeRecord->{$k};
}
// get field lists for source and/or target
if ($classTarget) {
$objData->fieldsTarget = $dc->activeRecord->targetType ? serialize($classTarget::getFieldList($dc->activeRecord, 'target')) : false;
}
if ($classSource) {
$objData->fieldsSource = $dc->activeRecord->sourceType ? serialize($classSource::getFieldList($dc->activeRecord, 'source')) : false;
}
// save the model
$objData->save();
return;
}