/**
* Forces a full reindex of all rows in the database or, optionally, a single table
*
* @param array $pa_table_name
* @param array $pa_options Reindexing options:
* showProgress
* interactiveProgressDisplay
* log
* callback
*/
public function reindex($pa_table_names = null, $pa_options = null)
{
define('__CollectiveAccess_IS_REINDEXING__', 1);
$t_timer = new Timer();
$pb_display_progress = isset($pa_options['showProgress']) ? (bool) $pa_options['showProgress'] : true;
$pb_interactive_display = isset($pa_options['interactiveProgressDisplay']) ? (bool) $pa_options['interactiveProgressDisplay'] : false;
$ps_callback = isset($pa_options['callback']) ? (string) $pa_options['callback'] : false;
if ($pa_table_names) {
if (!is_array($pa_table_names)) {
$pa_table_names = array($pa_table_names);
}
$va_table_names = array();
foreach ($pa_table_names as $vs_table) {
if ($this->opo_datamodel->tableExists($vs_table)) {
$vn_num = $this->opo_datamodel->getTableNum($vs_table);
print "\nTRUNCATING {$vs_table}\n\n";
$this->opo_engine->truncateIndex($vn_num);
$t_instance = $this->opo_datamodel->getInstanceByTableName($vs_table, true);
$va_table_names[$vn_num] = array('name' => $vs_table, 'num' => $vn_num, 'displayName' => $t_instance->getProperty('NAME_PLURAL'));
}
}
if (!sizeof($va_table_names)) {
return false;
}
} else {
// full reindex
$this->opo_engine->truncateIndex();
$va_table_names = $this->getIndexedTables();
}
$o_db = $this->opo_db;
if ($pb_display_progress || $ps_callback) {
$va_names = array();
foreach ($va_table_names as $vn_table_num => $va_table_info) {
$va_names[] = $va_table_info['displayName'];
}
if ($pb_display_progress) {
print "\nWILL INDEX [" . join(", ", $va_names) . "]\n\n";
}
}
$vn_tc = 0;
foreach ($va_table_names as $vn_table_num => $va_table_info) {
$vs_table = $va_table_info['name'];
$t_table_timer = new Timer();
$t_instance = $this->opo_datamodel->getInstanceByTableName($vs_table, true);
$vs_table_pk = $t_instance->primaryKey();
$va_fields_to_index = $this->getFieldsToIndex($vn_table_num);
if (!is_array($va_fields_to_index) || sizeof($va_fields_to_index) == 0) {
continue;
}
$qr_all = $o_db->query("SELECT " . $t_instance->primaryKey() . " FROM {$vs_table}");
$vn_num_rows = $qr_all->numRows();
if ($pb_display_progress) {
print CLIProgressBar::start($vn_num_rows, _t('Indexing %1', $t_instance->getProperty('NAME_PLURAL')));
}
$vn_c = 0;
while ($qr_all->nextRow()) {
$t_instance->load($qr_all->get($t_instance->primaryKey()));
$t_instance->doSearchIndexing(array(), true, $this->opo_engine->engineName());
if ($pb_display_progress && $pb_interactive_display) {
print CLIProgressBar::next();
}
if ($ps_callback && !($vn_c % 100)) {
$ps_callback($vn_c, $vn_num_rows, null, null, (double) $t_timer->getTime(2), memory_get_usage(true), $va_table_names, $vn_table_num, $t_instance->getProperty('NAME_PLURAL'), $vn_tc + 1);
}
$vn_c++;
}
$qr_all->free();
unset($t_instance);
if ($pb_display_progress && $pb_interactive_display) {
print CLIProgressBar::finish();
}
$this->opo_engine->optimizeIndex($vn_table_num);
$vn_tc++;
}
if ($pb_display_progress) {
print "\n\n\nDone! [Indexing for " . join(", ", $va_names) . " took " . caFormatInterval((double) $t_timer->getTime(4)) . "]\n";
}
if ($ps_callback) {
$ps_callback(1, 1, _t('Elapsed time: %1', caFormatInterval((double) $t_timer->getTime(2))), _t('Index rebuild complete!'), (double) $t_timer->getTime(2), memory_get_usage(true), $va_table_names, null, null, sizeof($va_table_names));
}
}
/**
* Forces a full reindex of all rows in the database or, optionally, a single table
*
* @param array $pa_table_names
* @param array $pa_options Reindexing options:
* showProgress
* interactiveProgressDisplay
* log
* callback
* @return null|false
*/
public function reindex($pa_table_names = null, $pa_options = null)
{
define('__CollectiveAccess_IS_REINDEXING__', 1);
$t_timer = new Timer();
$pb_display_progress = isset($pa_options['showProgress']) ? (bool) $pa_options['showProgress'] : true;
$pb_interactive_display = isset($pa_options['interactiveProgressDisplay']) ? (bool) $pa_options['interactiveProgressDisplay'] : false;
$ps_callback = isset($pa_options['callback']) ? (string) $pa_options['callback'] : false;
if ($pa_table_names) {
if (!is_array($pa_table_names)) {
$pa_table_names = array($pa_table_names);
}
$va_table_names = array();
foreach ($pa_table_names as $vs_table) {
if ($this->opo_datamodel->tableExists($vs_table)) {
$vn_num = $this->opo_datamodel->getTableNum($vs_table);
if ($pb_display_progress) {
print "\nTRUNCATING {$vs_table}\n\n";
}
$this->opo_engine->truncateIndex($vn_num);
$t_instance = $this->opo_datamodel->getInstanceByTableName($vs_table, true);
$va_table_names[$vn_num] = array('name' => $vs_table, 'num' => $vn_num, 'displayName' => $t_instance->getProperty('NAME_PLURAL'));
}
}
if (!sizeof($va_table_names)) {
return false;
}
} else {
// full reindex
$this->opo_engine->truncateIndex();
$va_table_names = $this->getIndexedTables();
}
$o_db = $this->opo_db;
if ($pb_display_progress || $ps_callback) {
$va_names = array();
foreach ($va_table_names as $vn_table_num => $va_table_info) {
$va_names[] = $va_table_info['displayName'];
}
if ($pb_display_progress) {
print "\nWILL INDEX [" . join(", ", $va_names) . "]\n\n";
}
}
$vn_tc = 0;
foreach ($va_table_names as $vn_table_num => $va_table_info) {
$vs_table = $va_table_info['name'];
$t_table_timer = new Timer();
$t_instance = $this->opo_datamodel->getInstanceByTableName($vs_table, true);
$vs_table_pk = $t_instance->primaryKey();
$vn_table_num = $t_instance->tableNum();
$va_fields_to_index = $this->getFieldsToIndex($vn_table_num);
if (!is_array($va_fields_to_index) || sizeof($va_fields_to_index) == 0) {
continue;
}
$qr_all = $o_db->query("SELECT " . $t_instance->primaryKey() . " FROM {$vs_table}");
$vn_num_rows = $qr_all->numRows();
if ($pb_display_progress) {
print CLIProgressBar::start($vn_num_rows, _t('Indexing %1', $t_instance->getProperty('NAME_PLURAL')));
}
$vn_c = 0;
$va_ids = $qr_all->getAllFieldValues($t_instance->primaryKey());
$va_element_ids = null;
if (method_exists($t_instance, "getApplicableElementCodes")) {
$va_element_ids = array_keys($t_instance->getApplicableElementCodes(null, false, false));
}
$vn_table_num = $t_instance->tableNum();
$vs_table_pk = $t_instance->primaryKey();
$va_field_data = array();
$va_intrinsic_list = $this->getFieldsToIndex($vs_table, $vs_table, array('intrinsicOnly' => true));
$va_intrinsic_list[$vs_table_pk] = array();
foreach ($va_ids as $vn_i => $vn_id) {
if (!($vn_i % 200)) {
// Pre-load attribute values for next 200 items to index; improves index performance
$va_id_slice = array_slice($va_ids, $vn_i, 200);
if ($va_element_ids) {
ca_attributes::prefetchAttributes($o_db, $vn_table_num, $va_id_slice, $va_element_ids);
}
$qr_field_data = $o_db->query("\n\t\t\t\t\t\tSELECT " . join(", ", array_keys($va_intrinsic_list)) . " \n\t\t\t\t\t\tFROM {$vs_table}\n\t\t\t\t\t\tWHERE {$vs_table_pk} IN (?)\t\n\t\t\t\t\t", array($va_id_slice));
$va_field_data = array();
while ($qr_field_data->nextRow()) {
$va_field_data[(int) $qr_field_data->get($vs_table_pk)] = $qr_field_data->getRow();
}
}
$this->indexRow($vn_table_num, $vn_id, $va_field_data[$vn_id], true, null, array(), array());
if ($pb_display_progress && $pb_interactive_display) {
CLIProgressBar::setMessage("Memory: " . caGetMemoryUsage());
print CLIProgressBar::next();
}
if ($ps_callback && !($vn_c % 100)) {
$ps_callback($vn_c, $vn_num_rows, null, null, (double) $t_timer->getTime(2), memory_get_usage(true), $va_table_names, $vn_table_num, $t_instance->getProperty('NAME_PLURAL'), $vn_tc + 1);
}
$vn_c++;
}
$qr_all->free();
unset($t_instance);
if ($pb_display_progress && $pb_interactive_display) {
print CLIProgressBar::finish();
}
$this->opo_engine->optimizeIndex($vn_table_num);
$vn_tc++;
}
if ($pb_display_progress) {
print "\n\n\nDone! [Indexing for " . join(", ", $va_names) . " took " . caFormatInterval((double) $t_timer->getTime(4)) . "]\n";
print "Note that if you're using an external search service like Apache Solr, the data may only now be sent to the actual service because it was buffered until now. So you still might have to wait a while for the script to finish.\n";
}
if ($ps_callback) {
$ps_callback(1, 1, _t('Elapsed time: %1', caFormatInterval((double) $t_timer->getTime(2))), _t('Index rebuild complete!'), (double) $t_timer->getTime(2), memory_get_usage(true), $va_table_names, null, null, sizeof($va_table_names));
}
}
/**
*
*/
public static function regenerate_annotation_previews($po_opts = null)
{
require_once __CA_LIB_DIR__ . "/core/Db.php";
require_once __CA_MODELS_DIR__ . "/ca_representation_annotations.php";
$o_db = new Db();
$t_rep = new ca_object_representations();
$t_rep->setMode(ACCESS_WRITE);
if (!($vn_start = (int) $po_opts->getOption('start_id'))) {
$vn_start = null;
}
if (!($vn_end = (int) $po_opts->getOption('end_id'))) {
$vn_end = null;
}
$vs_sql_where = null;
$va_params = array();
if ($vn_start > 0 && $vn_end > 0 && $vn_start <= $vn_end || $vn_start > 0 && $vn_end == null) {
$vs_sql_where = "WHERE annotation_id >= ?";
$va_params[] = $vn_start;
if ($vn_end) {
$vs_sql_where .= " AND annotation_id <= ?";
$va_params[] = $vn_end;
}
}
$qr_reps = $o_db->query("\n\t\t\t\tSELECT annotation_id \n\t\t\t\tFROM ca_representation_annotations \n\t\t\t\t{$vs_sql_where}\n\t\t\t\tORDER BY annotation_id\n\t\t\t", $va_params);
$vn_total = $qr_reps->numRows();
print CLIProgressBar::start($vn_total, _t('Finding annotations'));
$vn_c = 1;
while ($qr_reps->nextRow()) {
$t_instance = new ca_representation_annotations($vn_id = $qr_reps->get('annotation_id'));
print CLIProgressBar::next(1, _t('Annotation %1', $vn_id));
$t_instance->setMode(ACCESS_WRITE);
$t_instance->update(array('forcePreviewGeneration' => true));
$vn_c++;
}
print CLIProgressBar::finish();
}
/**
* Set the current progress bar message
*
* @param string $ps_message The message to set.
* @param bool $pb_refresh Force the progress bar to display the new message. Default is true.
*
* @return bool True if message was set, false if not.
*/
public function setMessage($ps_message, $pb_refresh = true, $pa_options = null)
{
$this->ops_message = $ps_message;
$this->setCache($ps_message);
switch ($vs_mode = $this->getMode()) {
case 'CLI':
CLIProgressBar::setMessage($ps_message, $pa_options);
break;
case 'WebUI':
// noop
break;
}
if ($pb_refresh) {
$this->redraw();
}
return true;
}
/**
* Export a record set as defined by the given search expression and the table_num for this exporter.
* This function wraps the record-level exports using the settings 'wrap_before' and 'wrap_after' if they are set.
* @param string $ps_exporter_code defines the exporter to use
* @param SearchResult $po_result An existing SearchResult object
* @param string $ps_filename Destination filename (we can't keep everything in memory here)
* @param array $pa_options
* progressCallback = callback function for asynchronous UI status reporting
* showCLIProgressBar = Show command-line progress bar. Default is false.
* logDirectory = path to directory where logs should be written
* logLevel = KLogger constant for minimum log level to record. Default is KLogger::INFO. Constants are, in descending order of shrillness:
* KLogger::EMERG = Emergency messages (system is unusable)
* KLogger::ALERT = Alert messages (action must be taken immediately)
* KLogger::CRIT = Critical conditions
* KLogger::ERR = Error conditions
* KLogger::WARN = Warnings
* KLogger::NOTICE = Notices (normal but significant conditions)
* KLogger::INFO = Informational messages
* KLogger::DEBUG = Debugging messages
* @return boolean success state
*/
public static function exportRecordsFromSearchResult($ps_exporter_code, $po_result, $ps_filename, $pa_options = array())
{
if (!$po_result instanceof SearchResult) {
return false;
}
$vs_log_dir = caGetOption('logDirectory', $pa_options);
if (!file_exists($vs_log_dir) || !is_writable($vs_log_dir)) {
$vs_log_dir = caGetTempDirPath();
}
if (!($vn_log_level = caGetOption('logLevel', $pa_options))) {
$vn_log_level = KLogger::INFO;
}
$o_log = new KLogger($vs_log_dir, $vn_log_level);
ca_data_exporters::$s_exporter_cache = array();
ca_data_exporters::$s_exporter_item_cache = array();
$vb_show_cli_progress_bar = isset($pa_options['showCLIProgressBar']) && $pa_options['showCLIProgressBar'];
$po_request = caGetOption('request', $pa_options, null);
$vb_have_request = $po_request instanceof RequestHTTP;
if (!($t_mapping = ca_data_exporters::loadExporterByCode($ps_exporter_code))) {
return false;
}
$va_errors = ca_data_exporters::checkMapping($ps_exporter_code);
if (sizeof($va_errors) > 0) {
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, 0, -1, _t('Export failed: %1', join("; ", $va_errors)), 0, memory_get_usage(true), 0);
}
return false;
}
$o_log->logInfo(_t("Starting SearchResult-based multi-record export for mapping %1.", $ps_exporter_code));
$vn_start_time = time();
$vs_wrap_before = $t_mapping->getSetting('wrap_before');
$vs_wrap_after = $t_mapping->getSetting('wrap_after');
$t_instance = $t_mapping->getAppDatamodel()->getInstanceByTableNum($t_mapping->get('table_num'));
$vn_num_items = $po_result->numHits();
$o_log->logInfo(_t("SearchResult contains %1 results. Now calling single-item export for each record.", $vn_num_items));
if ($vs_wrap_before) {
file_put_contents($ps_filename, $vs_wrap_before . "\n", FILE_APPEND);
}
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::start($vn_num_items, _t('Processing search result'));
}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
if ($vn_num_items > 0) {
$ps_callback($po_request, 0, $vn_num_items, _t("Exporting result"), time() - $vn_start_time, memory_get_usage(true), 0);
} else {
$ps_callback($po_request, 0, -1, _t('Found no records to export'), time() - $vn_start_time, memory_get_usage(true), 0);
}
}
$vn_num_processed = 0;
if ($t_mapping->getSetting('CSV_print_field_names')) {
$va_header = $va_header_sources = array();
$va_mapping_items = $t_mapping->getItems();
foreach ($va_mapping_items as $vn_i => $va_mapping_item) {
$va_settings = caUnserializeForDatabase($va_mapping_item['settings']);
$va_header_sources[(int) $va_mapping_item['element']] = $va_settings['_id'] ? $va_settings['_id'] : $va_mapping_item['source'];
}
ksort($va_header_sources);
foreach ($va_header_sources as $vn_element => $vs_source) {
$va_tmp = explode(".", $vs_source);
if ($t_table = $t_mapping->getAppDatamodel()->getInstanceByTableName($va_tmp[0], true)) {
$va_header[] = $t_table->getDisplayLabel($vs_source);
} else {
$va_header[] = $vs_source;
}
}
file_put_contents($ps_filename, join(",", $va_header) . "\n", FILE_APPEND);
}
$i = 0;
while ($po_result->nextHit()) {
// clear caches every once in a while. doesn't make much sense to keep them around while exporting
if (++$i % 1000 == 0) {
SearchResult::clearCaches();
ca_data_exporters::clearCaches();
}
if ($vb_have_request) {
if (!caCanRead($po_request->getUserID(), $t_instance->tableNum(), $po_result->get($t_instance->primaryKey()))) {
continue;
}
}
$vs_item_export = ca_data_exporters::exportRecord($ps_exporter_code, $po_result->get($t_instance->primaryKey()), array('logger' => $o_log));
file_put_contents($ps_filename, $vs_item_export . "\n", FILE_APPEND);
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::next(1, _t("Exporting records ..."));
}
$vn_num_processed++;
if ($vb_have_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $vn_num_processed, $vn_num_items, _t("Exporting ... [%1/%2]", $vn_num_processed, $vn_num_items), time() - $vn_start_time, memory_get_usage(true), $vn_num_processed);
}
}
if ($vs_wrap_after) {
file_put_contents($ps_filename, $vs_wrap_after . "\n", FILE_APPEND);
}
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::finish();
}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $vn_num_items, $vn_num_items, _t('Export completed'), time() - $vn_start_time, memory_get_usage(true), $vn_num_processed);
}
return true;
}
/**
*
*
* @param string $ps_source
* @param string $ps_mapping
* @param array $pa_options
* user_id = user to execute import for
* description = Text describing purpose of import to be logged.
* showCLIProgressBar = Show command-line progress bar. Default is false.
* format = Format of data being imported. MANDATORY
* useNcurses = Use ncurses library to format output
* logDirectory = path to directory where logs should be written
* logLevel = KLogger constant for minimum log level to record. Default is KLogger::INFO. Constants are, in descending order of shrillness:
* KLogger::EMERG = Emergency messages (system is unusable)
* KLogger::ALERT = Alert messages (action must be taken immediately)
* KLogger::CRIT = Critical conditions
* KLogger::ERR = Error conditions
* KLogger::WARN = Warnings
* KLogger::NOTICE = Notices (normal but significant conditions)
* KLogger::INFO = Informational messages
* KLogger::DEBUG = Debugging messages
* dryRun = do import but don't actually save data
* debug = output tons of debugging information
*/
public static function importDataFromSource($ps_source, $ps_mapping, $pa_options = null)
{
ca_data_importers::$s_num_import_errors = 0;
ca_data_importers::$s_num_records_processed = 0;
ca_data_importers::$s_num_records_skipped = 0;
ca_data_importers::$s_import_error_list = array();
$va_notices = $va_errors = array();
if (!($t_mapping = ca_data_importers::mappingExists($ps_mapping))) {
return null;
}
$o_event = ca_data_import_events::newEvent(isset($pa_options['user_id']) ? $pa_options['user_id'] : null, $pa_options['format'], $ps_source, isset($pa_options['description']) ? $pa_options['description'] : '');
$o_trans = new Transaction();
$t_mapping->setTransaction($o_trans);
$po_request = caGetOption('request', $pa_options, null);
$pb_dry_run = caGetOption('dryRun', $pa_options, false);
$pb_debug = caGetOption('debug', $pa_options, false);
$o_config = Configuration::load();
if (!is_array($pa_options) || !isset($pa_options['logLevel']) || !$pa_options['logLevel']) {
$pa_options['logLevel'] = KLogger::INFO;
}
if (!is_array($pa_options) || !isset($pa_options['logDirectory']) || !$pa_options['logDirectory'] || !file_exists($pa_options['logDirectory'])) {
if (!($pa_options['logDirectory'] = $o_config->get('batch_metadata_import_log_directory'))) {
$pa_options['logDirectory'] = ".";
}
}
if (!is_writeable($pa_options['logDirectory'])) {
$pa_options['logDirectory'] = caGetTempDirPath();
}
$o_log = new KLogger($pa_options['logDirectory'], $pa_options['logLevel']);
$vb_show_cli_progress_bar = isset($pa_options['showCLIProgressBar']) && $pa_options['showCLIProgressBar'] ? true : false;
if ($vb_use_ncurses = isset($pa_options['useNcurses']) && $pa_options['useNcurses'] ? true : false) {
$vb_use_ncurses = caCLIUseNcurses();
}
$vb_use_ncurses = false;
$vn_error_window_height = null;
if ($vb_use_ncurses) {
$r_ncurse = ncurses_init();
$r_screen = ncurses_newwin(0, 0, 0, 0);
ncurses_border(0, 0, 0, 0, 0, 0, 0, 0);
ncurses_getmaxyx($r_screen, $vn_max_y, $vn_max_x);
$vn_error_window_height = $vn_max_y - 8;
$r_errors = ncurses_newwin($vn_error_window_height, $vn_max_x - 4, 4, 2);
ncurses_wborder($r_errors, 0, 0, 0, 0, 0, 0, 0, 0);
ncurses_wattron($r_errors, NCURSES_A_REVERSE);
ncurses_mvwaddstr($r_errors, 0, 1, _t(" Recent errors "));
ncurses_wattroff($r_errors, NCURSES_A_REVERSE);
$r_progress = ncurses_newwin(3, $vn_max_x - 4, $vn_max_y - 4, 2);
ncurses_wborder($r_progress, 0, 0, 0, 0, 0, 0, 0, 0);
ncurses_wattron($r_progress, NCURSES_A_REVERSE);
ncurses_mvwaddstr($r_progress, 0, 1, _t(" Progress "));
ncurses_wattroff($r_progress, NCURSES_A_REVERSE);
$r_status = ncurses_newwin(3, $vn_max_x - 4, 1, 2);
ncurses_wborder($r_status, 0, 0, 0, 0, 0, 0, 0, 0);
ncurses_wattron($r_status, NCURSES_A_REVERSE);
ncurses_mvwaddstr($r_status, 0, 1, _t(" Import status "));
ncurses_wattroff($r_status, NCURSES_A_REVERSE);
ncurses_refresh();
ncurses_wrefresh($r_progress);
ncurses_wrefresh($r_errors);
ncurses_wrefresh($r_status);
}
$o_log->logInfo(_t('Started import of %1 using mapping %2', $ps_source, $t_mapping->get("importer_code")));
$va_log_import_error_opts = array('window' => $r_errors, 'log' => $o_log, 'request' => $po_request, 'progressCallback' => isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback']) ? $ps_callback : null, 'reportCallback' => isset($pa_options['reportCallback']) && ($ps_callback = $pa_options['reportCallback']) ? $ps_callback : null);
global $g_ui_locale_id;
// constant locale set by index.php for web requests
$vn_locale_id = isset($pa_options['locale_id']) && (int) $pa_options['locale_id'] ? (int) $pa_options['locale_id'] : $g_ui_locale_id;
$o_dm = $t_mapping->getAppDatamodel();
$t = new Timer();
$vn_start_time = time();
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::start(0, _t('Reading %1', $ps_source), array('window' => $r_progress));
}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, 0, 100, _t('Reading %1', $ps_source), time() - $vn_start_time, memory_get_usage(true), 0, ca_data_importers::$s_num_import_errors);
}
// Open file
$ps_format = isset($pa_options['format']) && $pa_options['format'] ? $pa_options['format'] : null;
if (!($o_reader = $t_mapping->getDataReader($ps_source, $ps_format))) {
ca_data_importers::logImportError(_t("Could not open source %1 (format=%2)", $ps_source, $ps_format), $va_log_import_error_opts);
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
if (!$o_reader->read($ps_source)) {
ca_data_importers::logImportError(_t("Could not read source %1 (format=%2)", $ps_source, $ps_format), $va_log_import_error_opts);
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
$o_log->logDebug(_t('Finished reading input source at %1 seconds', $t->getTime(4)));
$vn_num_items = $o_reader->numRows();
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::start($vn_num_items, _t('Importing from %1', $ps_source), array('window' => $r_progress));
}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, 0, $vn_num_items, _t('Importing from %1', $ps_source), time() - $vn_start_time, memory_get_usage(true), 0, ca_data_importers::$s_num_import_errors);
}
// What are we importing?
$vn_table_num = $t_mapping->get('table_num');
if (!($t_subject = $o_dm->getInstanceByTableNum($vn_table_num))) {
// invalid table
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
$t_subject->setTransaction($o_trans);
$vs_subject_table_name = $t_subject->tableName();
$t_label = $t_subject->getLabelTableInstance();
$t_label->setTransaction($o_trans);
$vs_label_display_fld = $t_subject->getLabelDisplayField();
$vs_subject_table = $t_subject->tableName();
$vs_type_id_fld = $t_subject->getTypeFieldName();
$vs_idno_fld = $t_subject->getProperty('ID_NUMBERING_ID_FIELD');
// get mapping rules
$va_mapping_rules = $t_mapping->getRules();
// get mapping groups
$va_mapping_groups = $t_mapping->getGroups();
$va_mapping_items = $t_mapping->getItems();
//
// Mapping-level settings
//
$vs_type_mapping_setting = $t_mapping->getSetting('type');
$vn_num_initial_rows_to_skip = $t_mapping->getSetting('numInitialRowsToSkip');
if (!in_array($vs_import_error_policy = $t_mapping->getSetting('errorPolicy'), array('ignore', 'stop'))) {
$vs_import_error_policy = 'ignore';
}
if (!in_array($vs_existing_record_policy = $t_mapping->getSetting('existingRecordPolicy'), array('none', 'skip_on_idno', 'skip_on_preferred_labels', 'merge_on_idno', 'merge_on_preferred_labels', 'merge_on_idno_and_preferred_labels', 'merge_on_idno_with_replace', 'merge_on_preferred_labels_with_replace', 'merge_on_idno_and_preferred_labels_with_replace', 'overwrite_on_idno', 'overwrite_on_preferred_labels', 'overwrite_on_idno_and_preferred_labels'))) {
$vs_existing_record_policy = 'none';
}
// Analyze mapping for figure out where type, idno, preferred label and other mandatory fields are coming from
$vn_type_id_mapping_item_id = $vn_idno_mapping_item_id = null;
$va_preferred_label_mapping_ids = array();
$va_mandatory_field_mapping_ids = array();
$va_mandatory_fields = $t_subject->getMandatoryFields();
foreach ($va_mapping_items as $vn_item_id => $va_item) {
$vs_destination = $va_item['destination'];
if (sizeof($va_dest_tmp = explode(".", $vs_destination)) >= 2) {
if ($va_dest_tmp[0] == $vs_subject_table && $va_dest_tmp[1] == 'preferred_labels') {
if (isset($va_dest_tmp[2])) {
$va_preferred_label_mapping_ids[$vn_item_id] = $va_dest_tmp[2];
} else {
$va_preferred_label_mapping_ids[$vn_item_id] = $vs_label_display_fld;
}
continue;
}
}
switch ($vs_destination) {
case "{$vs_subject_table}.{$vs_type_id_fld}":
$vn_type_id_mapping_item_id = $vn_item_id;
break;
case "{$vs_subject_table}.{$vs_idno_fld}":
$vn_idno_mapping_item_id = $vn_item_id;
break;
}
foreach ($va_mandatory_fields as $vs_mandatory_field) {
if ($vs_mandatory_field == $vs_type_id_fld) {
continue;
}
// type is handled separately
if ($vs_destination == "{$vs_subject_table}.{$vs_mandatory_field}") {
$va_mandatory_field_mapping_ids[$vs_mandatory_field] = $vn_item_id;
}
}
}
$va_items_by_group = array();
foreach ($va_mapping_items as $vn_item_id => $va_item) {
$va_items_by_group[$va_item['group_id']][$va_item['item_id']] = $va_item;
}
$o_log->logDebug(_t('Finished analyzing mapping at %1 seconds', $t->getTime(4)));
//
// Run through rows
//
$vn_row = 0;
ca_data_importers::$s_num_records_processed = 0;
while ($o_reader->nextRow()) {
$va_mandatory_field_values = array();
$vs_preferred_label_for_log = null;
if ($vn_row < $vn_num_initial_rows_to_skip) {
// skip over initial header rows
$vn_row++;
continue;
}
$vn_row++;
$t->startTimer();
$o_log->logDebug(_t('Started reading row %1 at %2 seconds', $vn_row, $t->getTime(4)));
$t_subject = $o_dm->getInstanceByTableNum($vn_table_num);
$t_subject->setTransaction($o_trans);
$t_subject->setMode(ACCESS_WRITE);
// Update status display
if ($vb_use_ncurses && ca_data_importers::$s_num_records_processed) {
ncurses_mvwaddstr($r_status, 1, 2, _t("Items processed/skipped: %1/%2", ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_records_skipped) . str_repeat(" ", 5) . _t("Errors: %1 (%2)", ca_data_importers::$s_num_import_errors, sprintf("%3.1f", ca_data_importers::$s_num_import_errors / ca_data_importers::$s_num_records_processed * 100) . "%") . str_repeat(" ", 6) . _t("Mapping: %1", $ps_mapping) . str_repeat(" ", 5) . _t("Source: %1", $ps_source) . str_repeat(" ", 5) . date("Y-m-d H:i:s") . str_repeat(" ", 5));
ncurses_refresh();
ncurses_wrefresh($r_status);
}
//
// Get data for current row
//
$va_row = $o_reader->getRow();
//
// Apply rules
//
foreach ($va_mapping_rules as $va_rule) {
if (!isset($va_rule['trigger']) || !$va_rule['trigger']) {
continue;
}
if (!isset($va_rule['actions']) || !is_array($va_rule['actions']) || !sizeof($va_rule['actions'])) {
continue;
}
$vm_ret = ExpressionParser::evaluate($va_rule['trigger'], $va_row);
if (!ExpressionParser::hadError() && (bool) $vm_ret) {
foreach ($va_rule['actions'] as $va_action) {
if (!is_array($va_action) && strtolower($va_action) == 'skip') {
$va_action = array('action' => 'skip');
}
switch ($vs_action_code = strtolower($va_action['action'])) {
case 'set':
$va_row[$va_action['target']] = $va_action['value'];
// TODO: transform value using mapping rules?
break;
case 'skip':
default:
if ($vs_action_code != 'skip') {
$o_log->logInfo(_t('Row was skipped using rule "%1" with default action because an invalid action ("%2") was specified', $va_rule['trigger'], $vs_action_code));
} else {
$o_log->logDebug(_t('Row was skipped using rule "%1" with action "%2"', $va_rule['trigger'], $vs_action_code));
}
continue 4;
break;
}
}
}
}
//
// Perform mapping and insert
//
// Get minimal info for imported row (type_id, idno, label)
// Get type
if ($vn_type_id_mapping_item_id) {
// Type is specified in row
$vs_type = ca_data_importers::getValueFromSource($va_mapping_items[$vn_type_id_mapping_item_id], $o_reader);
} else {
// Type is constant for all rows
$vs_type = $vs_type_mapping_setting;
}
// Get idno
$vs_idno = null;
if ($vn_idno_mapping_item_id) {
// idno is specified in row
$vs_idno = ca_data_importers::getValueFromSource($va_mapping_items[$vn_idno_mapping_item_id], $o_reader);
if (isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['default']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['default']) && !strlen($vs_idno)) {
$vs_idno = $va_mapping_items[$vn_idno_mapping_item_id]['settings']['default'];
}
if (!is_array($vs_idno) && $vs_idno[0] == '^' && preg_match("!^\\^[^ ]+\$!", $vs_idno)) {
// Parse placeholder when it's at the beginning of the value
if (!is_null($vm_parsed_val = BaseRefinery::parsePlaceholder($vs_idno, $va_row, $va_item, $vs_delimiter, $vn_c, array('returnAsString' => true)))) {
$vs_idno = $vm_parsed_val;
}
}
// Apply prefix/suffix *AFTER* setting default
if (isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix'])) {
$vs_idno = $va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix'] . $vs_idno;
}
if (isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix'])) {
$vs_idno .= $va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix'];
}
if (isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate'])) {
$vs_idno = caProcessTemplate($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate'], $va_row);
}
} else {
$vs_idno = "%";
}
$vb_idno_is_template = (bool) preg_match('![%]+!', $vs_idno);
// get preferred labels
$va_pref_label_values = array();
foreach ($va_preferred_label_mapping_ids as $vn_preferred_label_mapping_id => $vs_preferred_label_mapping_fld) {
$vs_label_val = ca_data_importers::getValueFromSource($va_mapping_items[$vn_preferred_label_mapping_id], $o_reader);
// If a template is specified format the label value with it so merge-on-preferred_label doesn't fail
if (isset($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate']) && strlen($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate'])) {
$vs_label_val = caProcessTemplate($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate'], $va_row);
}
$va_pref_label_values[$vs_preferred_label_mapping_fld] = $vs_label_val;
}
$vs_display_label = isset($va_pref_label_values[$vs_label_display_fld]) ? $va_pref_label_values[$vs_label_display_fld] : $vs_idno;
//
// Look for existing record?
//
if ($vs_existing_record_policy != 'none') {
switch ($vs_existing_record_policy) {
case 'skip_on_idno':
if (!$vb_idno_is_template && $t_subject->load(array($t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno))) {
$o_log->logInfo(_t('[%1] Skipped import because of existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy));
ca_data_importers::$s_num_records_skipped++;
continue 2;
// skip because idno matched
}
break;
case 'skip_on_preferred_labels':
if (is_array($va_ids = $t_subject->getIDsByLabel($va_pref_label_values, null, $vs_type)) && sizeof($va_ids)) {
$o_log->logInfo(_t('[%1] Skipped import because of existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy));
ca_data_importers::$s_num_records_skipped++;
continue 2;
// skip because label matched
}
break;
case 'merge_on_idno_and_preferred_labels':
case 'merge_on_idno':
case 'merge_on_idno_and_preferred_labels_with_replace':
case 'merge_on_idno_with_replace':
if (!$vb_idno_is_template && $t_subject->load(array($t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno))) {
$o_log->logInfo(_t('[%1] Merged with existing record matched on identifer by policy %2', $vs_idno, $vs_existing_record_policy));
break;
}
if ($vs_existing_record_policy == 'merge_on_idno') {
break;
}
// fall through if merge_on_idno_and_preferred_labels
// fall through if merge_on_idno_and_preferred_labels
case 'merge_on_preferred_labels':
case 'merge_on_preferred_labels_with_replace':
if (is_array($va_ids = $t_subject->getIDsByLabel($va_pref_label_values, null, $vs_type)) && sizeof($va_ids)) {
$t_subject->load($va_ids[0]);
$o_log->logInfo(_t('[%1] Merged with existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy));
}
break;
case 'overwrite_on_idno_and_preferred_labels':
case 'overwrite_on_idno':
if (!$vb_idno_is_template && $vs_idno && $t_subject->load(array($t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno))) {
$t_subject->setMode(ACCESS_WRITE);
$t_subject->delete(true, array('hard' => true));
if ($t_subject->numErrors()) {
ca_data_importers::logImportError(_t('[%1] Could not delete existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy));
// Don't stop?
} else {
$o_log->logInfo(_t('[%1] Overwrote existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy));
$t_subject->clear();
break;
}
}
if ($vs_existing_record_policy == 'overwrite_on_idno') {
break;
}
// fall through if overwrite_on_idno_and_preferred_labels
// fall through if overwrite_on_idno_and_preferred_labels
case 'overwrite_on_preferred_labels':
if (is_array($va_ids = $t_subject->getIDsByLabel($va_pref_label_values, null, $vs_type)) && sizeof($va_ids)) {
$t_subject->load($va_ids[0]);
$t_subject->setMode(ACCESS_WRITE);
$t_subject->delete(true, array('hard' => true));
if ($t_subject->numErrors()) {
ca_data_importers::logImportError(_t('[%1] Could not delete existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy));
// Don't stop?
} else {
$o_log->logInfo(_t('[%1] Overwrote existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy));
break;
}
$t_subject->clear();
}
break;
}
}
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::next(1, _t("Importing %1", $vs_idno), array('window' => $r_progress));
}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, ca_data_importers::$s_num_records_processed, $vn_num_items, _t("[%3/%4] Processing %1 (%2)", caTruncateStringWithEllipsis($vs_display_label, 50), $vs_idno, ca_data_importers::$s_num_records_processed, $vn_num_items), time() - $vn_start_time, memory_get_usage(true), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_import_errors);
}
$vb_output_subject_preferred_label = false;
$va_content_tree = array();
foreach ($va_items_by_group as $vn_group_id => $va_items) {
$va_group = $va_mapping_groups[$vn_group_id];
$vs_group_destination = $va_group['destination'];
$va_group_tmp = explode(".", $vs_group_destination);
if (sizeof($va_items) < 2 && sizeof($va_group_tmp) > 2) {
array_pop($va_group_tmp);
}
$vs_target_table = $va_group_tmp[0];
if (!($t_target = $o_dm->getInstanceByTableName($vs_target_table, true))) {
// Invalid target table
$o_log->logWarn(_t('[%1] Skipped group %2 because target %3 is invalid', $vs_idno, $vn_group_id, $vs_target_table));
continue;
}
$va_group_buf = array();
foreach ($va_items as $vn_item_id => $va_item) {
$va_vals = ca_data_importers::getValueFromSource($va_item, $o_reader, array('returnAsArray' => true));
$vn_c = -1;
foreach ($va_vals as $vn_i => $vm_val) {
$vn_c++;
if (isset($va_item['settings']['convertNewlinesToHTML']) && (bool) $va_item['settings']['convertNewlinesToHTML'] && is_string($vm_val)) {
$vm_val = nl2br($vm_val);
}
// Get location in content tree for addition of new content
$va_item_dest = explode(".", $va_item['destination']);
$vs_item_terminal = $va_item_dest[sizeof($va_item_dest) - 1];
if (isset($va_item['settings']['restrictToTypes']) && is_array($va_item['settings']['restrictToTypes']) && !in_array($vs_type, $va_item['settings']['restrictToTypes'])) {
$o_log->logInfo(_t('[%1] Skipped row %2 because of type restriction', $vs_idno, $vn_row));
continue 4;
}
if (isset($va_item['settings']['skipGroupIfExpression']) && strlen(trim($va_item['settings']['skipGroupIfExpression']))) {
if ($vm_ret = ExpressionParser::evaluate($va_item['settings']['skipGroupIfExpression'], $va_row)) {
$o_log->logInfo(_t('[%1] Skipped group %2 because expression %3 is true', $vs_idno, $vn_group_id, $va_item['settings']['skipGroupIfExpression']));
continue 3;
}
}
if (isset($va_item['settings']['skipIfEmpty']) && (bool) $va_item['settings']['skipIfEmpty'] && !strlen($vm_val)) {
$o_log->logInfo(_t('[%1] Skipped mapping because value for %3 is empty', $vs_idno, $vs_item_terminal));
continue 2;
}
if (isset($va_item['settings']['skipGroupIfEmpty']) && (bool) $va_item['settings']['skipGroupIfEmpty'] && !strlen($vm_val)) {
$o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 is empty', $vs_idno, $vn_group_id, $vs_item_terminal));
continue 3;
}
if (isset($va_item['settings']['skipGroupIfValue']) && is_array($va_item['settings']['skipGroupIfValue']) && strlen($vm_val) && in_array($vm_val, $va_item['settings']['skipGroupIfValue'])) {
$o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 matches value %4', $vs_idno, $vn_group_id, $vs_item_terminal, $vm_val));
continue 3;
}
if (isset($va_item['settings']['skipGroupIfNotValue']) && is_array($va_item['settings']['skipGroupIfNotValue']) && strlen($vm_val) && !in_array($vm_val, $va_item['settings']['skipGroupIfNotValue'])) {
$o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 matches is not in list of values', $vs_idno, $vn_group_id, $vs_item_terminal));
continue 3;
}
if (isset($va_item['settings']['skipRowIfEmpty']) && (bool) $va_item['settings']['skipRowIfEmpty'] && !strlen($vm_val)) {
$o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 is empty', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id));
continue 4;
}
if (isset($va_item['settings']['skipRowIfValue']) && is_array($va_item['settings']['skipRowIfValue']) && strlen($vm_val) && in_array($vm_val, $va_item['settings']['skipRowIfValue'])) {
$o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 matches value %5', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id));
continue 4;
}
if (isset($va_item['settings']['skipRowIfNotValue']) && is_array($va_item['settings']['skipRowIfNotValue']) && strlen($vm_val) && !in_array($vm_val, $va_item['settings']['skipRowIfNotValue'])) {
$o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 is not in list of values', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id, $vm_val));
continue 4;
}
if (isset($va_item['settings']['default']) && strlen($va_item['settings']['default']) && !strlen($vm_val)) {
$vm_val = $va_item['settings']['default'];
}
if (!is_array($vm_val) && $vm_val[0] == '^' && preg_match("!^\\^[^ ]+\$!", $vm_val)) {
// Parse placeholder when it's at the beginning of the value
if (!is_null($vm_parsed_val = BaseRefinery::parsePlaceholder($vm_val, $va_row, $va_item, $vs_delimiter, $vn_c, array('returnAsString' => true)))) {
$vm_val = $vm_parsed_val;
}
}
if ($vn_type_id_mapping_item_id && $vn_item_id == $vn_type_id_mapping_item_id) {
continue;
}
if ($vn_idno_mapping_item_id && $vn_item_id == $vn_idno_mapping_item_id) {
continue;
}
// Apply prefix/suffix *AFTER* setting default
if (isset($va_item['settings']['prefix']) && strlen($va_item['settings']['prefix'])) {
$vm_val = $va_item['settings']['prefix'] . $vm_val;
}
if (isset($va_item['settings']['suffix']) && strlen($va_item['settings']['suffix'])) {
$vm_val .= $va_item['settings']['suffix'];
}
if (isset($va_item['settings']['formatWithTemplate']) && strlen($va_item['settings']['formatWithTemplate'])) {
$vm_val = caProcessTemplate($va_item['settings']['formatWithTemplate'], array_replace($va_row, array((string) $va_item['source'] => ca_data_importers::replaceValue($vm_val, $va_item))));
$va_row[$va_item['source']] = $vm_val;
// copy formatted data into row so refineries can pick it up
}
// Get mapping error policy
$vb_item_error_policy_is_default = false;
if (!isset($va_item['settings']['errorPolicy']) || !in_array($vs_item_error_policy = $va_item['settings']['errorPolicy'], array('ignore', 'stop'))) {
$vs_item_error_policy = 'ignore';
$vb_item_error_policy_is_default = true;
}
//
if (isset($va_item['settings']['relationshipType']) && strlen($vs_rel_type = $va_item['settings']['relationshipType']) && $vs_target_table != $vs_subject_table) {
$va_group_buf[$vn_c]['_relationship_type'] = $vs_rel_type;
}
// Is it a constant value?
if (preg_match("!^_CONSTANT_:[\\d]+:(.*)!", $va_item['source'], $va_matches)) {
$va_group_buf[$vn_c][$vs_item_terminal] = $va_matches[1];
// Set it and go onto the next item
if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) {
$va_mandatory_field_values[$vs_k] = $vm_val;
}
continue;
}
// Perform refinery call (if required)
if (isset($va_item['settings']['refineries']) && is_array($va_item['settings']['refineries'])) {
foreach ($va_item['settings']['refineries'] as $vs_refinery) {
if (!$vs_refinery) {
continue;
}
if ($o_refinery = RefineryManager::getRefineryInstance($vs_refinery)) {
$va_refined_values = $o_refinery->refine($va_content_tree, $va_group, $va_item, $va_row, array('mapping' => $t_mapping, 'source' => $ps_source, 'subject' => $t_subject, 'locale_id' => $vn_locale_id, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row));
if (!$va_refined_values || is_array($va_refined_values) && !sizeof($va_refined_values)) {
continue 2;
}
if ($o_refinery->returnsMultipleValues()) {
foreach ($va_refined_values as $va_refined_value) {
$va_refined_value['_errorPolicy'] = $vs_item_error_policy;
$va_group_buf[$vn_c] = $va_refined_value;
$vn_c++;
}
} else {
$va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy;
$va_group_buf[$vn_c][$vs_item_terminal] = $va_refined_values;
}
if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) {
$va_mandatory_field_values[$vs_k] = $vm_val;
}
continue 2;
} else {
ca_data_importers::logImportError(_t('[%1] Invalid refinery %2 specified', $vs_idno, $vs_refinery));
}
}
}
if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) {
$va_mandatory_field_values[$vs_k] = $vm_val;
}
$vn_max_length = !is_array($vm_val) && isset($va_item['settings']['maxLength']) && (int) $va_item['settings']['maxLength'] ? (int) $va_item['settings']['maxLength'] : null;
if (isset($va_item['settings']['delimiter']) && strlen($vs_item_delimiter = $va_item['settings']['delimiter'])) {
$va_val_list = explode($vs_item_delimiter, $vm_val);
// Add delimited values
foreach ($va_val_list as $vs_list_val) {
$vs_list_val = trim(ca_data_importers::replaceValue($vs_list_val, $va_item));
if ($vn_max_length && mb_strlen($vs_list_val) > $vn_max_length) {
$vs_list_val = mb_substr($vs_list_val, 0, $vn_max_length);
}
$va_group_buf[$vn_c] = array($vs_item_terminal => $vs_list_val, '_errorPolicy' => $vs_item_error_policy);
$vn_c++;
}
$vn_row++;
continue;
// Don't add "regular" value below
}
if ($vn_max_length && mb_strlen($vm_val) > $vn_max_length) {
$vm_val = mb_substr($vm_val, 0, $vn_max_length);
}
switch ($vs_item_terminal) {
case 'preferred_labels':
case 'nonpreferred_labels':
if ($t_instance = $o_dm->getInstanceByTableName($vs_target_table, true)) {
$va_group_buf[$vn_c][$t_instance->getLabelDisplayField()] = $vm_val;
}
if (!$vb_item_error_policy_is_default || !isset($va_group_buf[$vn_c]['_errorPolicy'])) {
if (is_array($va_group_buf[$vn_c])) {
$va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy;
}
}
if ($vs_item_terminal == 'preferred_labels') {
$vs_preferred_label_for_log = $vm_val;
}
break;
default:
$va_group_buf[$vn_c][$vs_item_terminal] = $vm_val;
if (!$vb_item_error_policy_is_default || !isset($va_group_buf[$vn_c]['_errorPolicy'])) {
if (is_array($va_group_buf[$vn_c])) {
$va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy;
}
}
break;
}
}
// end foreach($va_vals as $vm_val)
}
foreach ($va_group_buf as $vn_i => $va_group_data) {
$va_ptr =& $va_content_tree;
foreach ($va_group_tmp as $vs_tmp) {
if (!is_array($va_ptr[$vs_tmp])) {
$va_ptr[$vs_tmp] = array();
}
$va_ptr =& $va_ptr[$vs_tmp];
if ($vs_tmp == $vs_target_table) {
// add numeric index after table to ensure repeat values don't overwrite each other
$va_parent =& $va_ptr;
$va_ptr[] = array();
$va_ptr =& $va_ptr[sizeof($va_ptr) - 1];
}
}
$va_ptr = $va_group_data;
}
}
//
// Process out self-relationships
//
if (is_array($va_content_tree[$vs_subject_table])) {
$va_self_related_content = array();
foreach ($va_content_tree[$vs_subject_table] as $vn_i => $va_element_data) {
if (isset($va_element_data['_relationship_type'])) {
$va_self_related_content[] = $va_element_data;
unset($va_content_tree[$vs_subject_table][$vn_i]);
}
}
if (sizeof($va_self_related_content) > 0) {
$va_content_tree["related.{$vs_subject_table}"] = $va_self_related_content;
}
}
$vn_row++;
$o_log->logDebug(_t('Finished building content tree for %1 at %2 seconds', $vs_idno, $t->getTime(4)));
$o_log->logDebug(_t("Content tree is\n%1", print_R($va_content_tree, true)));
//
// Process data in subject record
//
//print_r($va_content_tree);
//die("END\n\n");
//continue;
//if ($pb_debug && $po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
//$ps_callback($po_request, ca_data_importers::$s_num_records_processed, $vn_num_items, "<pre>".preg_replace("![\n\r\t ]+!", " ", print_r($va_content_tree, true))."</pre>", (time() - $vn_start_time), memory_get_usage(true), 0, ca_data_importers::$s_num_import_errors);
//}
if (!sizeof($va_content_tree) && !str_replace("%", "", $vs_idno)) {
continue;
}
if (!$t_subject->getPrimaryKey()) {
$o_event->beginItem($vn_row, $t_subject->tableNum(), 'I');
$t_subject->setMode(ACCESS_WRITE);
$t_subject->set($vs_type_id_fld, $vs_type);
if ($vb_idno_is_template) {
$t_subject->setIdnoWithTemplate($vs_idno);
} else {
$t_subject->set($vs_idno_fld, $vs_idno, array('assumeIdnoStubForLotID' => true));
// assumeIdnoStubForLotID forces ca_objects.lot_id values to always be considered as a potential idno_stub first, before use as a ca_objects.lot_di
}
foreach ($va_mandatory_field_mapping_ids as $vs_mandatory_field => $vn_mandatory_mapping_item_id) {
$t_subject->set($vs_mandatory_field, $va_mandatory_field_values[$vs_mandatory_field], array('assumeIdnoStubForLotID' => true));
}
$t_subject->insert();
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not insert new record"), array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
$o_trans->rollback();
return false;
}
continue;
}
$o_log->logDebug(_t('Created idno %1 at %2 seconds', $vs_idno, $t->getTime(4)));
} else {
$o_event->beginItem($vn_row_id, $t_subject->tableNum(), 'U');
// update
$t_subject->setMode(ACCESS_WRITE);
if ($vb_idno_is_template) {
$t_subject->setIdnoWithTemplate($vs_idno);
} else {
$t_subject->set($vs_idno_fld, $vs_idno, array('assumeIdnoStubForLotID' => true));
// assumeIdnoStubForLotID forces ca_objects.lot_id values to always be considered as a potential idno_stub first, before use as a ca_objects.lot_di
}
$t_subject->update();
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not update matched record"), array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
$o_trans->rollback();
return false;
}
continue;
}
$t_subject->clearErrors();
if (sizeof($va_preferred_label_mapping_ids) && $t_subject->getPreferredLabelCount() > 0) {
$t_subject->removeAllLabels(__CA_LABEL_TYPE_PREFERRED__);
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not update remove preferred labels from matched record"), array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
return false;
}
}
}
$o_log->logDebug(_t('Updated idno %1 at %2 seconds', $vs_idno, $t->getTime(4)));
}
$va_elements_set_for_this_record = array();
foreach ($va_content_tree as $vs_table_name => $va_content) {
if ($vs_table_name == $vs_subject_table) {
foreach ($va_content as $vn_i => $va_element_data) {
foreach ($va_element_data as $vs_element => $va_element_content) {
if (is_array($va_element_content)) {
$vs_item_error_policy = $va_element_content['_errorPolicy'];
unset($va_element_content['_errorPolicy']);
} else {
$vs_item_error_policy = null;
}
$t_subject->clearErrors();
$t_subject->setMode(ACCESS_WRITE);
switch ($vs_element) {
case 'preferred_labels':
$t_subject->addLabel($va_element_content, $vn_locale_id, isset($va_element_content['type_id']) ? $va_element_content['type_id'] : null, true);
if ($t_subject->numErrors() == 0) {
$vb_output_subject_preferred_label = true;
}
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add preferred label. Record was deleted because no preferred label could be applied: ", $vs_idno), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
$t_subject->delete(true, array('hard' => false));
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy %1', $vs_import_error_policy));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
$o_trans->rollback();
return false;
}
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
continue 5;
}
break;
case 'nonpreferred_labels':
$t_subject->addLabel($va_element_content, $vn_locale_id, isset($va_element_content['type_id']) ? $va_element_content['type_id'] : null, false);
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add non-preferred label:", $vs_idno), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
$o_trans->rollback();
return false;
}
}
break;
default:
if ($t_subject->hasField($vs_element)) {
$t_subject->set($vs_element, $va_element_content[$vs_element], array('assumeIdnoStubForLotID' => true));
$t_subject->update();
break;
}
if (is_array($va_element_content)) {
$va_element_content['locale_id'] = $vn_locale_id;
}
if (!isset($va_elements_set_for_this_record[$vs_element]) && !$va_elements_set_for_this_record[$vs_element] && in_array($vs_existing_record_policy, array('merge_on_idno_with_replace', 'merge_on_preferred_labels_with_replace', 'merge_on_idno_and_preferred_labels_with_replace'))) {
$t_subject->removeAttributes($vs_element, array('force' => true));
}
$va_elements_set_for_this_record[$vs_element] = true;
$t_subject->addAttribute($va_element_content, $vs_element, null, array('showRepeatCountErrors' => true, 'alwaysTreatValueAsIdno' => true));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Failed to add value for %2; values were %3: ", $vs_idno, $vs_element, ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
$o_trans->rollback();
return false;
}
}
$t_subject->update();
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Invalid %2; values were %3: ", $vs_idno, $vs_element, ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
$o_trans->rollback();
return false;
}
}
break;
}
}
}
} else {
// related
$vs_table_name = preg_replace('!^related\\.!', '', $vs_table_name);
foreach ($va_content as $vn_i => $va_element_data) {
$va_data_for_rel_table = $va_element_data;
$va_nonpreferred_labels = isset($va_data_for_rel_table['nonpreferred_labels']) ? $va_data_for_rel_table['nonpreferred_labels'] : null;
unset($va_data_for_rel_table['preferred_labels']);
unset($va_data_for_rel_table['_relationship_type']);
unset($va_data_for_rel_table['_type']);
unset($va_data_for_rel_table['_parent_id']);
$va_data_for_rel_table = array_merge($va_data_for_rel_table, ca_data_importers::_extractIntrinsicValues($va_data_for_rel_table, $vs_table_name));
switch ($vs_table_name) {
case 'ca_objects':
if ($vn_rel_id = DataMigrationUtils::getObjectID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related object with relationship %2", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_object_lots':
if ($vn_rel_id = DataMigrationUtils::getObjectLotID($va_element_data['idno_stub'], $va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno_stub']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related object lot with relationship %2", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_entities':
if ($vn_rel_id = DataMigrationUtils::getEntityID($va_element_data['preferred_labels'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related entity with relationship %2", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_places':
if ($vn_rel_id = DataMigrationUtils::getPlaceID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related place with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_collections':
if ($vn_rel_id = DataMigrationUtils::getCollectionID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, $vs_rel_type, null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related collection with relationship %2:", $vs_idno, $vs_rel_type), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_occurrences':
if ($vn_rel_id = DataMigrationUtils::getOccurrenceID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, $vs_rel_type, null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related occurrence with relationship %2:", $vs_idno, $vs_rel_type), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_storage_locations':
if ($vn_rel_id = DataMigrationUtils::getStorageLocationID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related storage location with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_list_items':
$va_data_for_rel_table['is_enabled'] = 1;
$va_data_for_rel_table['preferred_labels'] = $va_element_data['preferred_labels'];
if ($vn_rel_id = DataMigrationUtils::getListItemID($va_element_data['_list'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related list item with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_object_representations':
if ($vs_subject_table_name == 'ca_objects' && $va_element_data['media']['media']) {
$va_attributes = $va_element_data;
unset($va_attributes['media']);
if (!$t_subject->addRepresentation($va_element_data['media']['media'], isset($va_element_data['_type']) ? $va_element_data['_type'] : caGetDefaultItemID('object_representation_types'), $vn_locale_id, 0, 0, true, $va_attributes)) {
$vs_error = join("; ", $t_subject->getErrors());
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_loans':
if ($vn_rel_id = DataMigrationUtils::getLoanID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related loan with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
case 'ca_movements':
if ($vn_rel_id = DataMigrationUtils::getMovementID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('matchOn' => array('idno', 'label'), 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related movement with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_trans->rollback();
return false;
}
}
}
break;
}
if (is_array($va_element_data['_related_related']) && sizeof($va_element_data['_related_related'])) {
foreach ($va_element_data['_related_related'] as $vs_rel_rel_table => $va_rel_rels) {
foreach ($va_rel_rels as $vn_i => $va_rel_rel) {
if (!($t_rel_instance = $o_dm->getInstanceByTableName($vs_table_name))) {
$o_log->logWarn(_t("[%1] Could not instantiate related table %2", $vs_idno, $vs_table_name));
continue;
}
if ($t_rel_instance->load($vn_rel_id)) {
if ($t_rel_rel = $t_rel_instance->addRelationship($vs_rel_rel_table, $va_rel_rel['id'], $va_rel_rel['_relationship_type'])) {
$o_log->logInfo(_t('[%1] Related %2 (%3) to related %4 with relationship %5', $vs_idno, $o_dm->getTableProperty($vs_rel_rel_table, 'NAME_SINGULAR'), $va_rel_rel['id'], $t_rel_instance->getProperty('NAME_SINGULAR'), trim($va_rel_rel['_relationship_type'])));
} else {
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related %2 (%3) to related %4 with relationship %5:", $vs_idno, $o_dm->getTableProperty($vs_rel_rel_table, 'NAME_SINGULAR'), $va_rel_rel['id'], $t_rel_instance->getProperty('NAME_SINGULAR'), trim($va_rel_rel['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
}
}
}
}
}
}
}
}
}
// $t_subject->update();
//
// if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Invalid %2; values were %3: ", $vs_idno, 'attributes', ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
// ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
// if ($vs_item_error_policy == 'stop') {
// $o_log->logAlert(_t('Import stopped due to mapping error policy'));
// if($vb_use_ncurses) { ncurses_end(); }
//
// $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
//
// $o_trans->rollback();
// return false;
// }
// }
//
$o_log->logDebug(_t('Finished inserting content tree for %1 at %2 seconds into database', $vs_idno, $t->getTime(4)));
if (!$vb_output_subject_preferred_label && $t_subject->getPreferredLabelCount() == 0) {
$t_subject->addLabel(array($vs_label_display_fld => '???'), $vn_locale_id, null, true);
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add default label", $vs_idno), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
$o_trans->rollback();
return false;
}
}
}
$o_log->logInfo(_t('[%1] Imported %2 as %3 ', $vs_idno, $vs_preferred_label_for_log, $vs_subject_table_name));
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_SUCCESS__, _t('Imported %1', $vs_idno));
ca_data_importers::$s_num_records_processed++;
}
$o_log->logInfo(_t('Import of %1 completed using mapping %2: %3 imported/%4 skipped/%5 errors', $ps_source, $t_mapping->get('importer_code'), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_records_skipped, ca_data_importers::$s_num_import_errors));
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::finish();
}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $vn_num_items, $vn_num_items, _t('Import completed'), time() - $vn_start_time, memory_get_usage(true), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_import_errors);
}
if (isset($pa_options['reportCallback']) && ($ps_callback = $pa_options['reportCallback'])) {
$va_general = array('elapsedTime' => time() - $vn_start_time, 'numErrors' => ca_data_importers::$s_num_import_errors, 'numProcessed' => ca_data_importers::$s_num_records_processed);
$ps_callback($po_request, $va_general, ca_data_importers::$s_import_error_list, true);
}
if ($vb_use_ncurses) {
ncurses_end();
}
if ($pb_dry_run) {
$o_trans->rollback();
$o_log->logInfo(_t('Rollback successful import run in "dry run" mode'));
} else {
$o_trans->commit();
}
return true;
}
public static function reload_service_values($po_opts = null)
{
$va_infoservice_elements = ca_metadata_elements::getElementsAsList(false, null, null, true, false, false, array(__CA_ATTRIBUTE_VALUE_INFORMATIONSERVICE__));
$o_db = new Db();
foreach ($va_infoservice_elements as $va_element) {
$qr_values = $o_db->query("\n\t\t\t\t\tSELECT * FROM ca_attribute_values\n\t\t\t\t\tWHERE element_id = ?\n\t\t\t\t", $va_element['element_id']);
print CLIProgressBar::start($qr_values->numRows(), "Reloading values for element code " . $va_element['element_code']);
$t_value = new ca_attribute_values();
while ($qr_values->nextRow()) {
$o_val = new InformationServiceAttributeValue($qr_values->getRow());
$vs_uri = $o_val->getUri();
print CLIProgressBar::next();
// inc before first continuation point
if (!$vs_uri || !strlen($vs_uri)) {
continue;
}
if (!$t_value->load($qr_values->get('value_id'))) {
continue;
}
$t_value->editValue($vs_uri);
if ($t_value->numErrors() > 0) {
print _t('There were errors updating an attribute row: ') . join(' ', $t_value->getErrors());
}
}
print CLIProgressBar::finish();
}
return true;
}
/**
* Set the width the rendered text must fit in
*
* @param int $width passed in options
*
* @static
* @return void
*/
protected static function setWidth($width = null)
{
if ($width === null) {
if (self::$window) {
ncurses_getmaxyx(self::$window, $vn_max_y, $vn_max_x);
$width = $vn_max_x - 4;
} else {
if (DIRECTORY_SEPARATOR === '/') {
$width = `tput cols`;
}
}
if ($width < 80) {
$width = 80;
}
}
self::$width = $width;
}
/**
*
*/
public static function reload_object_current_locations($po_opts = null)
{
require_once __CA_LIB_DIR__ . "/core/Db.php";
require_once __CA_MODELS_DIR__ . "/ca_objects.php";
$o_db = new Db();
$t_object = new ca_objects();
$qr_res = $o_db->query("SELECT * FROM ca_objects");
print CLIProgressBar::start($qr_res->numRows(), _t('Starting...'));
$vn_c = 0;
while ($qr_res->nextRow()) {
$vn_object_id = $qr_res->get('object_id');
if ($t_object->load($vn_object_id)) {
print CLIProgressBar::next(1, _t('Processing %1', $t_object->getWithTemplate("^ca_objects.preferred_labels.name (^ca_objects.idno)")));
$t_object->deriveCurrentLocationForBrowse();
} else {
print CLIProgressBar::next(1, _t('Cannot load object %1', $vn_object_id));
}
$vn_c++;
}
print CLIProgressBar::finish();
CLIUtils::addMessage(_t('Processed %1 objects', $vn_c));
return true;
}
/**
* Set the width the rendered text must fit in
*
* @param int $width passed in options
*
* @static
* @return void
*/
protected static function setWidth($width = null)
{
$height = null;
if ($width === null) {
if (self::$window) {
ncurses_getmaxyx(self::$window, $width, $height);
$width = 96;
} else {
if (DIRECTORY_SEPARATOR === '/') {
$width = `tput cols`;
}
}
if ($width < 80) {
$width = 80;
}
}
self::$width = $width;
}
/**
* Export a record set as defined by the given search expression and the table_num for this exporter.
* This function wraps the record-level exports using the settings 'wrap_before' and 'wrap_after' if they are set.
* @param string $ps_exporter_code defines the exporter to use
* @param string $ps_expression A valid search expression
* @param string $ps_filename Destination filename (we can't keep everything in memory here)
* @param array $pa_options
* showCLIProgressBar = Show command-line progress bar. Default is false.
* @return boolean success state
*/
public static function exportRecordsFromSearchExpression($ps_exporter_code, $ps_expression, $ps_filename, $pa_options = array())
{
ca_data_exporters::$s_exporter_cache = array();
ca_data_exporters::$s_exporter_item_cache = array();
$vb_show_cli_progress_bar = isset($pa_options['showCLIProgressBar']) && $pa_options['showCLIProgressBar'];
$po_request = isset($pa_options['request']) ? $pa_options['request'] : null;
if (!($t_mapping = ca_data_exporters::loadExporterByCode($ps_exporter_code))) {
return false;
}
$va_errors = ca_data_exporters::checkMapping($ps_exporter_code);
if (sizeof($va_errors) > 0) {
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, 0, -1, _t('Export failed: %1', join("; ", $va_errors)), 0, memory_get_usage(true), 0);
}
return false;
}
$vn_start_time = time();
$vs_wrap_before = $t_mapping->getSetting('wrap_before');
$vs_wrap_after = $t_mapping->getSetting('wrap_after');
$t_instance = $t_mapping->getAppDatamodel()->getInstanceByTableNum($t_mapping->get('table_num'));
$o_search = caGetSearchInstance($t_mapping->get('table_num'));
$o_result = $o_search->search($ps_expression);
$vn_num_items = $o_result->numHits();
if ($vs_wrap_before) {
file_put_contents($ps_filename, $vs_wrap_before . "\n", FILE_APPEND);
}
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::start($vn_num_items, _t('Processing search result'));
}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
if ($vn_num_items > 0) {
$ps_callback($po_request, 0, $vn_num_items, _t("Exporting result for search expression '%1'", $ps_expression), time() - $vn_start_time, memory_get_usage(true), 0);
} else {
$ps_callback($po_request, 0, -1, _t('Found no records to export'), time() - $vn_start_time, memory_get_usage(true), 0);
}
}
$vn_num_processed = 0;
while ($o_result->nextHit()) {
$vs_item_export = ca_data_exporters::exportRecord($ps_exporter_code, $o_result->get($t_instance->primaryKey()));
file_put_contents($ps_filename, $vs_item_export . "\n", FILE_APPEND);
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::next(1, _t("Exporting records ..."));
}
$vn_num_processed++;
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $vn_num_processed, $vn_num_items, _t("Exporting ... [%1/%2]", $vn_num_processed, $vn_num_items), time() - $vn_start_time, memory_get_usage(true), $vn_num_processed);
}
}
if ($vs_wrap_after) {
file_put_contents($ps_filename, $vs_wrap_after . "\n", FILE_APPEND);
}
if ($vb_show_cli_progress_bar) {
print CLIProgressBar::finish();
}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $vn_num_items, $vn_num_items, _t('Export completed'), time() - $vn_start_time, memory_get_usage(true), $vn_num_processed);
}
return true;
}
public static function reload_object_current_location_dates($po_opts = null)
{
require_once __CA_MODELS_DIR__ . "/ca_movements.php";
require_once __CA_MODELS_DIR__ . "/ca_movements_x_objects.php";
require_once __CA_MODELS_DIR__ . "/ca_movements_x_storage_locations.php";
$o_config = Configuration::load();
$o_db = new Db();
// Reload movements-objects
if ($vs_movement_storage_element = $o_config->get('movement_storage_location_date_element')) {
$qr_movements = ca_movements::find(['deleted' => 0], ['returnAs' => 'searchResult']);
print CLIProgressBar::start($qr_movements->numHits(), "Reloading movement dates");
while ($qr_movements->nextHit()) {
if ($va_dates = $qr_movements->get("ca_movements.{$vs_movement_storage_element}", ['returnAsArray' => true, 'rawDate' => true])) {
$va_date = array_shift($va_dates);
// get movement-object relationships
if (is_array($va_rel_ids = $qr_movements->get('ca_movements_x_objects.relation_id', ['returnAsArray' => true])) && sizeof($va_rel_ids)) {
$qr_res = $o_db->query("UPDATE ca_movements_x_objects SET sdatetime = ?, edatetime = ? WHERE relation_id IN (?)", array($va_date['start'], $va_date['end'], $va_rel_ids));
}
// get movement-location relationships
if (is_array($va_rel_ids = $qr_movements->get('ca_movements_x_storage_locations.relation_id', ['returnAsArray' => true])) && sizeof($va_rel_ids)) {
$qr_res = $o_db->query("UPDATE ca_movements_x_storage_locations SET sdatetime = ?, edatetime = ? WHERE relation_id IN (?)", array($va_date['start'], $va_date['end'], $va_rel_ids));
// check to see if archived storage locations are set in ca_movements_x_storage_locations.source_info
// Databases created prior to the October 2015 location tracking changes won't have this
$qr_rels = caMakeSearchResult('ca_movements_x_storage_locations', $va_rel_ids);
while ($qr_rels->nextHit()) {
if (!is_array($va_source_info = $qr_rels->get('source_info')) || !isset($va_source_info['path'])) {
$vn_rel_id = $qr_rels->get('ca_movements_x_storage_locations.relation_id');
$qr_res = $o_db->query("UPDATE ca_movements_x_storage_locations SET source_info = ? WHERE relation_id = ?", array(caSerializeForDatabase(array('path' => $qr_rels->get('ca_storage_locations.hierarchy.preferred_labels.name', array('returnAsArray' => true)), 'ids' => $qr_rels->get('ca_storage_locations.hierarchy.location_id', array('returnAsArray' => true)))), $vn_rel_id));
}
}
}
print CLIProgressBar::next();
}
}
print CLIProgressBar::finish();
}
return true;
}
