/**
* Process import object
*
* @return array
*/
public function process()
{
$result = ['success' => false, 'error' => [], 'hint' => []];
if (empty($this->import_data)) {
throw new Exception('You must pecify import_data parameter.');
}
// if we have fixes to the data
if (method_exists($this, 'overrides')) {
$this->overrides();
}
// initialize alias & crypt objects
$alias_object = new object_data_aliases();
$alias_data = $alias_object->get();
$crypt = new crypt();
gc_enable();
// processing one by one
foreach (array_keys($this->import_data) as $k) {
// we continue if we have no rows
if (count($this->import_data[$k]['data']) == 0) {
continue;
}
// class and object
$class = $this->import_data[$k]['options']['model'];
$object = new $class();
// a short cut to skip updating large datasets
if (!empty($this->import_data[$k]['options']['quick_pk_comparison'])) {
// data from an array
$groupped = [];
foreach ($this->import_data[$k]['data'] as $k12 => $v12) {
$keys = [];
foreach ($this->import_data[$k]['options']['quick_pk_comparison'] as $v13) {
$keys[] = $v12[$v13];
}
$keys = implode('::', $keys);
$temp = array_key_get($groupped, $keys);
if (empty($temp)) {
$temp = 0;
}
$temp++;
array_key_set($groupped, $keys, $temp);
}
// get data from database
$sql = "SELECT concat_ws('::', " . implode(', ', $this->import_data[$k]['options']['quick_pk_comparison']) . ") groupped, count(*) count FROM {$object->name} GROUP BY groupped";
$db = $object->db_object();
$result_compare = $db->query($sql);
$groupped2 = [];
foreach ($result_compare['rows'] as $v12) {
$groupped2[$v12['groupped']] = $v12['count'];
}
// compare
$discrepancies = false;
foreach ($groupped as $k12 => $v12) {
if (!isset($groupped2[$k12]) || $groupped2[$k12] != $v12) {
$discrepancies = true;
}
}
// we continue loop if there's no discrepancies
if (!$discrepancies) {
$result['hint'][] = ' * Skipping ' . $object->name . ', db link: ' . $object->db_link;
continue;
}
}
// if we need mass import
$total_rows = count($this->import_data[$k]['data']);
if ($total_rows >= self::mass_import_rows) {
$db = $object->db_object();
$db->create_temp_table('temp_' . $object->name, $object->columns, $this->import_data[$k]['options']['pk'], ['skip_serials' => true]);
}
$counter = 0;
$buffer = [];
do {
// grab first element from the array
$v2 = array_shift($this->import_data[$k]['data']);
// we need to process overrides
foreach ($v2 as $k3 => $v3) {
if (!is_string($v3)) {
continue;
}
// if we need id
if (strpos($v3, '~id~') === 0) {
$value = substr($v3, 4);
$alias = null;
foreach ($alias_data as $k4 => $v4) {
// todo: maybe need column prefix with alias
if (strpos($k3, $k4) !== false) {
$alias = $k4;
}
}
$v2[$k3] = $alias_object->get_id_by_code($alias, substr($v3, 4));
continue;
}
// password
if (strpos($v3, '~password~') === 0) {
$v2[$k3] = $crypt->password_hash(substr($v3, 10));
}
}
// if we have multiple
if (!empty($this->import_data[$k]['options']['multiple'])) {
foreach ($v2[$this->import_data[$k]['options']['multiple'][0]] as $mv0) {
$temp = $v2;
$temp[$this->import_data[$k]['options']['multiple'][0]] = $mv0;
if (!empty($this->import_data[$k]['options']['multiple'][1])) {
foreach ($v2[$this->import_data[$k]['options']['multiple'][1]] as $mv1) {
$temp[$this->import_data[$k]['options']['multiple'][1]] = $mv1;
// todo: add third level
}
}
$buffer[] = $temp;
}
} else {
$buffer[] = $v2;
}
// if buffer has 100 rows or we have no data
if (count($buffer) > 249 || count($buffer) > 0 && count($this->import_data[$k]['data']) == 0) {
if ($total_rows >= self::mass_import_rows) {
// insert all rows
$result_insert = $db->insert('temp_' . $object->name, $buffer);
if (!$result_insert['success']) {
$result['error'] = $result_insert['error'];
return $result;
}
$counter += count($buffer);
// doing this might take some time
echo ".";
} else {
// less than 1000 records
foreach ($buffer as $v10) {
switch ($this->import_data[$k]['options']['method'] ?? 'save') {
case 'save_insert_new':
$result_insert = $object->save($v10, ['pk' => $this->import_data[$k]['options']['pk'], 'flag_insert_only' => true, 'ignore_not_set_fields' => true]);
break;
case 'save':
default:
$result_insert = $object->save($v10, ['pk' => $this->import_data[$k]['options']['pk'], 'ignore_not_set_fields' => true]);
}
if (!$result_insert['success']) {
$result['error'] = $result_insert['error'];
return $result;
}
$counter++;
}
}
$buffer = [];
// free up memory
gc_collect_cycles();
}
} while (count($this->import_data[$k]['data']) > 0);
// we need to run few queries for mass import
if ($total_rows >= self::mass_import_rows) {
$type = $this->import_data[$k]['options']['method'] ?? 'save';
$columns = [];
$columns_mysql = [];
$where = [];
$where_mysql = [];
$where_delete = [];
$serials = false;
foreach ($object->columns as $k12 => $v12) {
if (in_array($k12, $this->import_data[$k]['options']['pk'])) {
$where[] = "a.{$k12} = temp_{$object->name}.{$k12}";
$where_mysql[] = "{$object->name}.{$k12} = temp_{$object->name}.{$k12}";
$where_delete[] = "a.{$k12} = b.{$k12}";
} else {
if (strpos($v12['type'], 'serial') !== false) {
$serials = true;
continue;
}
$columns[] = "{$k12} = temp_{$object->name}.{$k12}";
$columns_mysql[] = "{$object->name}.{$k12} = temp_{$object->name}.{$k12}";
}
}
if ($type == 'save') {
// update existing rows
if ($db->backend == 'pgsql') {
$sql = "UPDATE {$object->name} AS a SET " . implode(', ', $columns) . " FROM temp_{$object->name} AS temp_{$object->name} WHERE " . implode(' AND ', $where);
} else {
$sql = "UPDATE {$object->name}, temp_{$object->name} SET " . implode(', ', $columns_mysql) . " WHERE " . implode(' AND ', $where_mysql);
}
$result_insert = $db->query($sql);
if (!$result_insert['success']) {
$result['error'] = $result_insert['error'];
return $result;
}
}
// delete existing rows
if ($db->backend == 'pgsql') {
$sql = "DELETE FROM temp_{$object->name} a WHERE EXISTS (SELECT 1 FROM {$object->name} b WHERE " . implode(' AND ', $where_delete) . ")";
} else {
$sql = "DELETE FROM a USING temp_{$object->name} AS a INNER JOIN {$object->name} AS b WHERE " . implode(' AND ', $where_delete);
}
$result_insert = $db->query($sql);
if (!$result_insert['success']) {
$result['error'] = $result_insert['error'];
return $result;
}
// we need to update serial columns
if (!empty($serials)) {
$columns = [];
foreach ($object->columns as $k12 => $v12) {
if (strpos($v12['type'], 'serial') !== false) {
$columns[] = "{$k12} = nextval('{$object->name}_{$k12}_seq')";
}
}
$sql = "UPDATE temp_{$object->name} SET " . implode(', ', $columns);
$result_insert = $db->query($sql);
if (!$result_insert['success']) {
$result['error'] = $result_insert['error'];
return $result;
}
}
// insert the rest of the rows
$sql = "INSERT INTO {$object->name} SELECT * FROM temp_{$object->name}";
$result_insert = $db->query($sql);
if (!$result_insert['success']) {
$result['error'] = $result_insert['error'];
return $result;
}
// drop temp table
$sql = "DROP TABLE temp_{$object->name}";
$result_insert = $db->query($sql);
if (!$result_insert['success']) {
$result['error'] = $result_insert['error'];
return $result;
}
}
$result['hint'][] = ' * Imported ' . $counter . ' rows into ' . $object->name . ', db link: ' . $object->db_link;
}
$result['success'] = true;
return $result;
}
