* GNU General Public License for more details. * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ header("Content-Type: text/plain"); header("Cache-Control: no-cache, must-revalidate"); header("Expires: Sat, 26 Jul 1997 05:00:00 GMT"); function logAndDie($log, $message, $code) { http_response_code($code); $log->logFatal($message); die($message); } require_once 'KLogger.php'; $log = new KLogger('.', KLogger::INFO); $log->logDebug('Opened connection from ' . $_SERVER['REMOTE_ADDR']); $password = ''; $remoteaddr = ''; if ($_REQUEST['password'] != $password || $_SERVER['REMOTE_ADDR'] != $remoteaddr) { logAndDie($log, 'Bad Authentication: ' . $_SERVER['REMOTE_ADDR'], 403); } $log->logDebug('Authenticated.'); require_once 'mysql_settings.php'; $mysqli = new mysqli(mysqlSettings::HOSTNAME, mysqlSettings::USERNAME, mysqlSettings::PASSWORD, mysqlSettings::DATABASE); if ($mysqli->connect_error) { logAndDie($log, 'MySQL Connection Error', 500); } $log->logDebug('Connected to MySQL - Request mode: ' . $_REQUEST['mode']); if ($_REQUEST['mode'] === 'update') { $query = sprintf("UPDATE `logs_users` SET `ip` = '%s' WHERE `name` = '%s'", $mysqli->real_escape_string($_REQUEST['ip']), strtolower($mysqli->real_escape_string($_REQUEST['name']))); $mysqli->query($query) or logAndDie($log, 'MySQL Query Error: ' . $query, 500);
/** * Export a single record using the mapping defined by this exporter and return as string * @param string $ps_exporter_code defines the exporter to use * @param int $pn_record_id Primary key of the record to export. Record type is determined by the table_num field for this exporter. * @param array $pa_options * singleRecord = Gives a signal to the export format implementation that this is a single record export. For certain formats * this might trigger different behavior, for instance the XML export format prepends the item-level output with <?xml ... ?> * in those cases. * rdfMode = Signals the implementation that this is an RDF mode export * logDirectory = path to directory where logs should be written * logLevel = KLogger constant for minimum log level to record. Default is KLogger::INFO. Constants are, in descending order of shrillness: * KLogger::EMERG = Emergency messages (system is unusable) * KLogger::ALERT = Alert messages (action must be taken immediately) * KLogger::CRIT = Critical conditions * KLogger::ERR = Error conditions * KLogger::WARN = Warnings * KLogger::NOTICE = Notices (normal but significant conditions) * KLogger::INFO = Informational messages * KLogger::DEBUG = Debugging messages * logger = Optional ready-to-use instance of KLogger to use for logging/debugging * @return string Exported record as string */ public static function exportRecord($ps_exporter_code, $pn_record_id, $pa_options = array()) { // The variable cache is valid for the whole record export. // It's being modified in ca_data_exporters::processExporterItem // and then reset here if we move on to the next record. ca_data_exporters::$s_variables = array(); $o_log = caGetOption('logger', $pa_options); // only set up new logging facilities if no existing one has been passed down if (!$o_log || !$o_log instanceof KLogger) { $vs_log_dir = caGetOption('logDirectory', $pa_options); if (!file_exists($vs_log_dir) || !is_writable($vs_log_dir)) { $vs_log_dir = caGetTempDirPath(); } if (!($vn_log_level = caGetOption('logLevel', $pa_options))) { $vn_log_level = KLogger::INFO; } $o_log = new KLogger($vs_log_dir, $vn_log_level); } // make sure we pass logger to item processor $pa_options['logger'] = $o_log; ca_data_exporters::$s_instance_cache = array(); $t_exporter = ca_data_exporters::loadExporterByCode($ps_exporter_code); if (!$t_exporter) { $o_log->logError(_t("Failed to load exporter with code '%1' for item with ID %2", $ps_exporter_code, $pn_record_id)); return false; } $o_log->logInfo(_t("Successfully loaded exporter with code '%1' for item with ID %2", $ps_exporter_code, $pn_record_id)); $va_export = array(); foreach ($t_exporter->getTopLevelItems() as $va_item) { $va_export = array_merge($va_export, $t_exporter->processExporterItem($va_item['item_id'], $t_exporter->get('table_num'), $pn_record_id, $pa_options)); } $o_log->logInfo(_t("The export tree for exporter code '%1' and item with ID %2 is now ready to be processed by the export format (i.e. transformed to XML, for example).", $ps_exporter_code, $pn_record_id)); $o_log->logDebug(print_r($va_export, true)); // we may wanna auto-load this? switch ($t_exporter->getSetting('exporter_format')) { case 'XML': $o_export = new ExportXML(); break; case 'MARC': $o_export = new ExportMARC(); break; case 'CSV': $o_export = new ExportCSV(); break; case 'ExifTool': $o_export = new ExportExifTool(); break; default: return; } $o_export->setLogger($o_log); // if someone wants to mangle the whole tree ... well, go right ahead $o_manager = new ApplicationPluginManager(); $o_manager->hookExportRecord(array('exporter_instance' => $t_exporter, 'record_id' => $pn_record_id, 'export' => &$va_export)); $pa_options['settings'] = $t_exporter->getSettings(); $vs_wrap_before = $t_exporter->getSetting('wrap_before_record'); $vs_wrap_after = $t_exporter->getSetting('wrap_after_record'); if ($vs_wrap_before || $vs_wrap_after) { $pa_options['singleRecord'] = false; } $vs_export = $o_export->processExport($va_export, $pa_options); if (strlen($vs_wrap_before) > 0) { $vs_export = $vs_wrap_before . "\n" . $vs_export; } if (strlen($vs_wrap_after) > 0) { $vs_export = $vs_export . "\n" . $vs_wrap_after; } return $vs_export; }
/** * * * @param string $ps_source * @param string $ps_mapping * @param array $pa_options * user_id = user to execute import for * description = Text describing purpose of import to be logged. * showCLIProgressBar = Show command-line progress bar. Default is false. * format = Format of data being imported. MANDATORY * useNcurses = Use ncurses library to format output * logDirectory = path to directory where logs should be written * logLevel = KLogger constant for minimum log level to record. Default is KLogger::INFO. Constants are, in descending order of shrillness: * KLogger::EMERG = Emergency messages (system is unusable) * KLogger::ALERT = Alert messages (action must be taken immediately) * KLogger::CRIT = Critical conditions * KLogger::ERR = Error conditions * KLogger::WARN = Warnings * KLogger::NOTICE = Notices (normal but significant conditions) * KLogger::INFO = Informational messages * KLogger::DEBUG = Debugging messages * dryRun = do import but don't actually save data * environment = an array of environment values to provide to the import process. The keys manifest themselves as mappable tags. * forceImportForPrimaryKeys = list of primary key ids to force mapped source data into. The number of keys passed should equal or exceed the number of rows in the source data. [Default is empty] * transaction = transaction to perform import within. Will not be used if noTransaction option is set. [Default is to create a new transaction] * noTransaction = don't wrap the import in a transaction. [Default is false] */ public static function importDataFromSource($ps_source, $ps_mapping, $pa_options = null) { ca_data_importers::$s_num_import_errors = 0; ca_data_importers::$s_num_records_processed = 0; ca_data_importers::$s_num_records_skipped = 0; ca_data_importers::$s_import_error_list = array(); $opa_app_plugin_manager = new ApplicationPluginManager(); $va_notices = $va_errors = array(); $pb_no_transaction = caGetOption('noTransaction', $pa_options, false, array('castTo' => 'bool')); $pa_force_import_for_primary_keys = caGetOption('forceImportForPrimaryKeys', $pa_options, null); if (!($t_mapping = ca_data_importers::mappingExists($ps_mapping))) { return null; } $o_event = ca_data_import_events::newEvent(isset($pa_options['user_id']) ? $pa_options['user_id'] : null, $pa_options['format'], $ps_source, isset($pa_options['description']) ? $pa_options['description'] : ''); $o_trans = null; if (!$pb_no_transaction) { if (!($o_trans = caGetOption('transaction', $pa_options, null))) { $o_trans = new Transaction(); } $t_mapping->setTransaction($o_trans); } $po_request = caGetOption('request', $pa_options, null); $pb_dry_run = caGetOption('dryRun', $pa_options, false); $pn_file_number = caGetOption('fileNumber', $pa_options, 0); $pn_number_of_files = caGetOption('numberOfFiles', $pa_options, 1); $o_config = Configuration::load(); if (!is_array($pa_options) || !isset($pa_options['logLevel']) || !$pa_options['logLevel']) { $pa_options['logLevel'] = KLogger::INFO; } if (!is_array($pa_options) || !isset($pa_options['logDirectory']) || !$pa_options['logDirectory'] || !file_exists($pa_options['logDirectory'])) { if (!($pa_options['logDirectory'] = $o_config->get('batch_metadata_import_log_directory'))) { $pa_options['logDirectory'] = "."; } } if (!is_writeable($pa_options['logDirectory'])) { $pa_options['logDirectory'] = caGetTempDirPath(); } $o_log = new KLogger($pa_options['logDirectory'], $pa_options['logLevel']); $vb_show_cli_progress_bar = isset($pa_options['showCLIProgressBar']) && $pa_options['showCLIProgressBar'] ? true : false; $o_progress = caGetOption('progressBar', $pa_options, new ProgressBar('WebUI')); if ($vb_show_cli_progress_bar) { $o_progress->setMode('CLI'); $o_progress->set('outputToTerminal', true); } if ($vb_use_ncurses = isset($pa_options['useNcurses']) && $pa_options['useNcurses'] ? true : false) { $vb_use_ncurses = caCLIUseNcurses(); } $vn_error_window_height = null; $vn_max_x = $vn_max_y = null; if ($vb_use_ncurses) { ncurses_init(); $r_screen = ncurses_newwin(0, 0, 0, 0); ncurses_border(0, 0, 0, 0, 0, 0, 0, 0); ncurses_getmaxyx($r_screen, $vn_max_y, $vn_max_x); $vn_error_window_height = $vn_max_y - 8; $r_errors = ncurses_newwin($vn_error_window_height, $vn_max_x - 4, 4, 2); ncurses_wborder($r_errors, 0, 0, 0, 0, 0, 0, 0, 0); ncurses_wattron($r_errors, NCURSES_A_REVERSE); ncurses_mvwaddstr($r_errors, 0, 1, _t(" Recent errors ")); ncurses_wattroff($r_errors, NCURSES_A_REVERSE); $r_progress = ncurses_newwin(3, $vn_max_x - 4, $vn_max_y - 4, 2); ncurses_wborder($r_progress, 0, 0, 0, 0, 0, 0, 0, 0); ncurses_wattron($r_progress, NCURSES_A_REVERSE); ncurses_mvwaddstr($r_progress, 0, 1, _t(" Progress ")); ncurses_wattroff($r_progress, NCURSES_A_REVERSE); $r_status = ncurses_newwin(3, $vn_max_x - 4, 1, 2); ncurses_wborder($r_status, 0, 0, 0, 0, 0, 0, 0, 0); ncurses_wattron($r_status, NCURSES_A_REVERSE); ncurses_mvwaddstr($r_status, 0, 1, _t(" Import status ")); ncurses_wattroff($r_status, NCURSES_A_REVERSE); ncurses_refresh(0); ncurses_wrefresh($r_progress); ncurses_wrefresh($r_errors); ncurses_wrefresh($r_status); } $o_log->logInfo(_t('Started import of %1 using mapping %2', $ps_source, $t_mapping->get("importer_code"))); $t = new Timer(); $vn_start_time = time(); $va_log_import_error_opts = array('startTime' => $vn_start_time, 'window' => $r_errors, 'log' => $o_log, 'request' => $po_request, 'progressCallback' => isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback']) ? $ps_callback : null, 'reportCallback' => isset($pa_options['reportCallback']) && ($ps_callback = $pa_options['reportCallback']) ? $ps_callback : null); global $g_ui_locale_id; // constant locale set by index.php for web requests $vn_locale_id = isset($pa_options['locale_id']) && (int) $pa_options['locale_id'] ? (int) $pa_options['locale_id'] : $g_ui_locale_id; $o_dm = $t_mapping->getAppDatamodel(); $o_progress->start(_t('Reading %1', $ps_source), array('window' => $r_progress)); if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) { $ps_callback($po_request, $pn_file_number, $pn_number_of_files, $ps_source, 0, 100, _t('Reading %1', $ps_source), time() - $vn_start_time, memory_get_usage(true), 0, ca_data_importers::$s_num_import_errors); } // Open file $ps_format = isset($pa_options['format']) && $pa_options['format'] ? $pa_options['format'] : null; if (!($o_reader = $t_mapping->getDataReader($ps_source, $ps_format))) { ca_data_importers::logImportError(_t("Could not open source %1 (format=%2)", $ps_source, $ps_format), $va_log_import_error_opts); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } if (!$o_reader->read($ps_source, array('basePath' => $t_mapping->getSetting('basePath')))) { ca_data_importers::logImportError(_t("Could not read source %1 (format=%2)", $ps_source, $ps_format), $va_log_import_error_opts); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } $o_log->logDebug(_t('Finished reading input source at %1 seconds', $t->getTime(4))); $vn_num_items = $o_reader->numRows(); $o_progress->setTotal($vn_num_items); $o_progress->start(_t('Importing from %1', $ps_source), array('window' => $r_progress)); if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) { $ps_callback($po_request, $pn_file_number, $pn_number_of_files, $ps_source, 0, $vn_num_items, _t('Importing from %1', $ps_source), time() - $vn_start_time, memory_get_usage(true), 0, ca_data_importers::$s_num_import_errors); } // What are we importing? $vn_table_num = $t_mapping->get('table_num'); if (!($t_subject = $o_dm->getInstanceByTableNum($vn_table_num))) { // invalid table if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } $t_subject->setTransaction($o_trans); $vs_subject_table_name = $t_subject->tableName(); $t_label = $t_subject->getLabelTableInstance(); $t_label->setTransaction($o_trans); $vs_label_display_fld = $t_subject->getLabelDisplayField(); $vs_subject_table = $t_subject->tableName(); $vs_type_id_fld = $t_subject->getTypeFieldName(); $vs_idno_fld = $t_subject->getProperty('ID_NUMBERING_ID_FIELD'); // get mapping rules $va_mapping_rules = $t_mapping->getRules(); // get mapping groups $va_mapping_groups = $t_mapping->getGroups(); $va_mapping_items = $t_mapping->getItems(); // // Mapping-level settings // $vs_type_mapping_setting = $t_mapping->getSetting('type'); $vn_num_initial_rows_to_skip = $t_mapping->getSetting('numInitialRowsToSkip'); if (!in_array($vs_import_error_policy = $t_mapping->getSetting('errorPolicy'), array('ignore', 'stop'))) { $vs_import_error_policy = 'ignore'; } if (!in_array($vs_existing_record_policy = $t_mapping->getSetting('existingRecordPolicy'), array('none', 'skip_on_idno', 'skip_on_preferred_labels', 'merge_on_idno', 'merge_on_preferred_labels', 'merge_on_idno_and_preferred_labels', 'merge_on_idno_with_replace', 'merge_on_preferred_labels_with_replace', 'merge_on_idno_and_preferred_labels_with_replace', 'overwrite_on_idno', 'overwrite_on_preferred_labels', 'overwrite_on_idno_and_preferred_labels'))) { $vs_existing_record_policy = 'none'; } // Analyze mapping for figure out where type, idno, preferred label and other mandatory fields are coming from $vn_type_id_mapping_item_id = $vn_idno_mapping_item_id = null; $va_preferred_label_mapping_ids = array(); $va_mandatory_field_mapping_ids = array(); $va_mandatory_fields = $t_subject->getMandatoryFields(); foreach ($va_mapping_items as $vn_item_id => $va_item) { $vs_destination = $va_item['destination']; if (sizeof($va_dest_tmp = explode(".", $vs_destination)) >= 2) { if ($va_dest_tmp[0] == $vs_subject_table && $va_dest_tmp[1] == 'preferred_labels') { if (isset($va_dest_tmp[2])) { $va_preferred_label_mapping_ids[$vn_item_id] = $va_dest_tmp[2]; } else { $va_preferred_label_mapping_ids[$vn_item_id] = $vs_label_display_fld; } continue; } } switch ($vs_destination) { case 'representation_id': if ($vs_subject_table == 'ca_representation_annotations') { $vn_type_id_mapping_item_id = $vn_item_id; } break; case "{$vs_subject_table}.{$vs_type_id_fld}": $vn_type_id_mapping_item_id = $vn_item_id; break; case "{$vs_subject_table}.{$vs_idno_fld}": $vn_idno_mapping_item_id = $vn_item_id; break; } foreach ($va_mandatory_fields as $vs_mandatory_field) { if ($vs_mandatory_field == $vs_type_id_fld) { continue; } // type is handled separately if ($vs_destination == "{$vs_subject_table}.{$vs_mandatory_field}") { $va_mandatory_field_mapping_ids[$vs_mandatory_field] = $vn_item_id; } } } $va_items_by_group = array(); foreach ($va_mapping_items as $vn_item_id => $va_item) { $va_items_by_group[$va_item['group_id']][$va_item['item_id']] = $va_item; } $o_log->logDebug(_t('Finished analyzing mapping at %1 seconds', $t->getTime(4))); // // Set up environment // $va_environment = caGetOption('environment', $pa_options, array(), array('castTo' => 'array')); if (is_array($va_environment_config = $t_mapping->getEnvironment())) { foreach ($va_environment_config as $vn_i => $va_environment_item) { $va_env_tmp = explode("|", $va_environment_item['value']); if (!($o_env_reader = $t_mapping->getDataReader($ps_source, $ps_format))) { break; } if (!$o_env_reader->read($ps_source, array('basePath' => ''))) { break; } $o_env_reader->nextRow(); switch (sizeof($va_env_tmp)) { case 1: $vs_env_value = $o_env_reader->get($va_environment_item['value'], array('returnAsArray' => false)); break; case 2: $vn_seek = (int) $va_env_tmp[0]; $o_reader->seek($vn_seek > 0 ? $vn_seek - 1 : $vn_seek); $o_env_reader->nextRow(); $vs_env_value = $o_env_reader->get($va_env_tmp[1], array('returnAsArray' => false)); $o_reader->seek(0); break; default: $vs_env_value = $va_environment_item['value']; break; } $va_environment[$va_environment_item['name']] = $vs_env_value; } } // // Run through rows // $vn_row = 0; ca_data_importers::$s_num_records_processed = 0; while ($o_reader->nextRow()) { $va_mandatory_field_values = array(); $vs_preferred_label_for_log = null; if ($vn_row < $vn_num_initial_rows_to_skip) { // skip over initial header rows $vn_row++; continue; } $vn_row++; $t->startTimer(); $o_log->logDebug(_t('Started reading row %1 at %2 seconds', $vn_row, $t->getTime(4))); $t_subject = $o_dm->getInstanceByTableNum($vn_table_num); if ($o_trans) { $t_subject->setTransaction($o_trans); } $t_subject->setMode(ACCESS_WRITE); // Update status display if ($vb_use_ncurses && ca_data_importers::$s_num_records_processed) { ncurses_mvwaddstr($r_status, 1, 2, _t("Items processed/skipped: %1/%2", ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_records_skipped) . str_repeat(" ", 5) . _t("Errors: %1 (%2)", ca_data_importers::$s_num_import_errors, sprintf("%3.1f", ca_data_importers::$s_num_import_errors / ca_data_importers::$s_num_records_processed * 100) . "%") . str_repeat(" ", 6) . _t("Mapping: %1", $ps_mapping) . str_repeat(" ", 5) . _t("Source: %1", $ps_source) . str_repeat(" ", 5) . date("Y-m-d H:i:s") . str_repeat(" ", 5)); ncurses_refresh(0); ncurses_wrefresh($r_status); } // // Get data for current row // $va_row = array_merge($o_reader->getRow(), $va_environment); // // Apply rules // foreach ($va_mapping_rules as $va_rule) { if (!isset($va_rule['trigger']) || !$va_rule['trigger']) { continue; } if (!isset($va_rule['actions']) || !is_array($va_rule['actions']) || !sizeof($va_rule['actions'])) { continue; } $vm_ret = ExpressionParser::evaluate($va_rule['trigger'], $va_row); if (!ExpressionParser::hadError() && (bool) $vm_ret) { foreach ($va_rule['actions'] as $va_action) { if (!is_array($va_action) && strtolower($va_action) == 'skip') { $va_action = array('action' => 'skip'); } switch ($vs_action_code = strtolower($va_action['action'])) { case 'set': $va_row[$va_action['target']] = $va_action['value']; // TODO: transform value using mapping rules? break; case 'skip': default: if ($vs_action_code != 'skip') { $o_log->logInfo(_t('Row was skipped using rule "%1" with default action because an invalid action ("%2") was specified', $va_rule['trigger'], $vs_action_code)); } else { $o_log->logDebug(_t('Row was skipped using rule "%1" with action "%2"', $va_rule['trigger'], $vs_action_code)); } continue 4; break; } } } } // // Perform mapping and insert // // Get minimal info for imported row (type_id, idno, label) // Get type if ($vn_type_id_mapping_item_id) { // Type is specified in row $vs_type = ca_data_importers::getValueFromSource($va_mapping_items[$vn_type_id_mapping_item_id], $o_reader, array('environment' => $va_environment)); } else { // Type is constant for all rows $vs_type = $vs_type_mapping_setting; } // Get idno $vs_idno = null; if ($vn_idno_mapping_item_id) { // idno is specified in row $vs_idno = ca_data_importers::getValueFromSource($va_mapping_items[$vn_idno_mapping_item_id], $o_reader, array('environment' => $va_environment)); if (isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['default']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['default']) && !strlen($vs_idno)) { $vs_idno = $va_mapping_items[$vn_idno_mapping_item_id]['settings']['default']; } if (!is_array($vs_idno) && $vs_idno[0] == '^' && preg_match("!^\\^[^ ]+\$!", $vs_idno)) { // Parse placeholder when it's at the beginning of the value if (!is_null($vm_parsed_val = BaseRefinery::parsePlaceholder($vs_idno, $va_row, $va_item, null, array('reader' => $o_reader, 'returnAsString' => true)))) { $vs_idno = $vm_parsed_val; } } // Apply prefix/suffix *AFTER* setting default if ($vs_idno && isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix'])) { $vs_idno = $va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix'] . $vs_idno; } if ($vs_idno && isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix'])) { $vs_idno .= $va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix']; } if (isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate'])) { $vs_idno = caProcessTemplate($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate'], $va_row); } } else { $vs_idno = "%"; } $vb_idno_is_template = (bool) preg_match('![%]+!', $vs_idno); // get preferred labels $va_pref_label_values = array(); foreach ($va_preferred_label_mapping_ids as $vn_preferred_label_mapping_id => $vs_preferred_label_mapping_fld) { $vs_label_val = ca_data_importers::getValueFromSource($va_mapping_items[$vn_preferred_label_mapping_id], $o_reader, array('environment' => $va_environment)); // If a template is specified format the label value with it so merge-on-preferred_label doesn't fail if (isset($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate']) && strlen($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate'])) { $vs_label_val = caProcessTemplate($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate'], $va_row); } $va_pref_label_values[$vs_preferred_label_mapping_fld] = $vs_label_val; } $vs_display_label = isset($va_pref_label_values[$vs_label_display_fld]) ? $va_pref_label_values[$vs_label_display_fld] : $vs_idno; // // Look for existing record? // if (is_array($pa_force_import_for_primary_keys) && sizeof($pa_force_import_for_primary_keys) > 0) { $vn_id = array_shift($pa_force_import_for_primary_keys); if (!$t_subject->load($vn_id)) { $o_log->logInfo(_t('[%1] Skipped import because of forced primary key \'%1\' does not exist', $vn_id)); ca_data_importers::$s_num_records_skipped++; continue; // skip because primary key does not exist } } elseif ($vs_existing_record_policy != 'none') { switch ($vs_existing_record_policy) { case 'skip_on_idno': if (!$vb_idno_is_template) { $va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, $t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno, 'deleted' => 0), array('returnAs' => 'ids'))); if (is_array($va_ids) && sizeof($va_ids) > 0) { $o_log->logInfo(_t('[%1] Skipped import because of existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy)); ca_data_importers::$s_num_records_skipped++; continue 2; // skip because idno matched } } break; case 'skip_on_preferred_labels': $va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, 'preferred_labels' => $va_pref_label_values, 'deleted' => 0), array('returnAs' => 'ids'))); if (is_array($va_ids) && sizeof($va_ids) > 0) { $o_log->logInfo(_t('[%1] Skipped import because of existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy)); ca_data_importers::$s_num_records_skipped++; continue 2; // skip because label matched } break; case 'merge_on_idno_and_preferred_labels': case 'merge_on_idno': case 'merge_on_idno_and_preferred_labels_with_replace': case 'merge_on_idno_with_replace': if (!$vb_idno_is_template) { $va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, $t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno, 'deleted' => 0), array('returnAs' => 'ids'))); if (is_array($va_ids) && sizeof($va_ids) > 0) { $t_subject->load($va_ids[0]); $o_log->logInfo(_t('[%1] Merged with existing record matched on identifer by policy %2', $vs_idno, $vs_existing_record_policy)); break; } } if (in_array($vs_existing_record_policy, array('merge_on_idno', 'merge_on_idno_with_replace'))) { break; } // fall through if merge_on_idno_and_preferred_labels // fall through if merge_on_idno_and_preferred_labels case 'merge_on_preferred_labels': case 'merge_on_preferred_labels_with_replace': $va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, 'preferred_labels' => $va_pref_label_values, 'deleted' => 0), array('returnAs' => 'ids'))); if (is_array($va_ids) && sizeof($va_ids) > 0) { $t_subject->load($va_ids[0]); $o_log->logInfo(_t('[%1] Merged with existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy)); } break; case 'overwrite_on_idno_and_preferred_labels': case 'overwrite_on_idno': if (!$vb_idno_is_template && $vs_idno) { $va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, $t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno, 'deleted' => 0), array('returnAs' => 'ids'))); if (is_array($va_ids) && sizeof($va_ids) > 0) { $t_subject->load($va_ids[0]); $t_subject->setMode(ACCESS_WRITE); $t_subject->delete(true, array('hard' => true)); if ($t_subject->numErrors()) { ca_data_importers::logImportError(_t('[%1] Could not delete existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy)); // Don't stop? } else { $o_log->logInfo(_t('[%1] Overwrote existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy)); $t_subject->clear(); break; } } } if ($vs_existing_record_policy == 'overwrite_on_idno') { break; } // fall through if overwrite_on_idno_and_preferred_labels // fall through if overwrite_on_idno_and_preferred_labels case 'overwrite_on_preferred_labels': $va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, 'preferred_labels' => $va_pref_label_values, 'deleted' => 0), array('returnAs' => 'ids'))); if (is_array($va_ids) && sizeof($va_ids) > 0) { $t_subject->load($va_ids[0]); $t_subject->setMode(ACCESS_WRITE); $t_subject->delete(true, array('hard' => true)); if ($t_subject->numErrors()) { ca_data_importers::logImportError(_t('[%1] Could not delete existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy)); // Don't stop? } else { $o_log->logInfo(_t('[%1] Overwrote existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy)); break; } $t_subject->clear(); } break; } } $o_progress->next(_t("Importing %1", $vs_idno), array('window' => $r_progress)); if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) { $ps_callback($po_request, $pn_file_number, $pn_number_of_files, $ps_source, ca_data_importers::$s_num_records_processed, $vn_num_items, _t("[%3/%4] Processing %1 (%2)", caTruncateStringWithEllipsis($vs_display_label, 50), $vs_idno, ca_data_importers::$s_num_records_processed, $vn_num_items), time() - $vn_start_time, memory_get_usage(true), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_import_errors); } $vb_output_subject_preferred_label = false; $va_content_tree = array(); foreach ($va_items_by_group as $vn_group_id => $va_items) { $va_group = $va_mapping_groups[$vn_group_id]; $vs_group_destination = $va_group['destination']; $va_group_tmp = explode(".", $vs_group_destination); if (sizeof($va_items) < 2 && sizeof($va_group_tmp) > 2) { array_pop($va_group_tmp); } $vs_target_table = $va_group_tmp[0]; if (!($t_target = $o_dm->getInstanceByTableName($vs_target_table, true))) { // Invalid target table $o_log->logWarn(_t('[%1] Skipped group %2 because target %3 is invalid', $vs_idno, $vn_group_id, $vs_target_table)); continue; } if ($o_trans) { $t_target->setTransaction($o_trans); } $va_group_buf = array(); foreach ($va_items as $vn_item_id => $va_item) { if ($vb_use_as_single_value = caGetOption('useAsSingleValue', $va_item['settings'], false)) { // Force repeating values to be imported as a single value $va_vals = array(ca_data_importers::getValueFromSource($va_item, $o_reader, array('delimiter' => caGetOption('delimiter', $va_item['settings'], ''), 'returnAsArray' => false))); } else { $va_vals = ca_data_importers::getValueFromSource($va_item, $o_reader, array('returnAsArray' => true, 'environment' => $va_environment)); } if (!sizeof($va_vals)) { $va_vals = array(0 => null); } // consider missing values equivalent to blanks // Do value conversions foreach ($va_vals as $vn_i => $vm_val) { if (isset($va_item['settings']['default']) && strlen($va_item['settings']['default']) && !strlen($vm_val)) { $vm_val = $va_item['settings']['default']; } // Apply prefix/suffix *AFTER* setting default if ($vm_val && isset($va_item['settings']['prefix']) && strlen($va_item['settings']['prefix'])) { $vm_val = $va_item['settings']['prefix'] . $vm_val; } if ($vm_val && isset($va_item['settings']['suffix']) && strlen($va_item['settings']['suffix'])) { $vm_val .= $va_item['settings']['suffix']; } if (!is_array($vm_val) && $vm_val[0] == '^' && preg_match("!^\\^[^ ]+\$!", $vm_val)) { // Parse placeholder if (!is_null($vm_parsed_val = BaseRefinery::parsePlaceholder($vm_val, $va_row, $va_item, $vn_i, array('reader' => $o_reader, 'returnAsString' => true)))) { $vm_val = $vm_parsed_val; } } if (isset($va_item['settings']['formatWithTemplate']) && strlen($va_item['settings']['formatWithTemplate'])) { $vm_val = caProcessTemplate($va_item['settings']['formatWithTemplate'], array_replace($va_row, array((string) $va_item['source'] => ca_data_importers::replaceValue($vm_val, $va_item))), array('getFrom' => $o_reader)); } if (isset($va_item['settings']['applyRegularExpressions']) && is_array($va_item['settings']['applyRegularExpressions'])) { if (is_array($va_item['settings']['applyRegularExpressions'])) { foreach ($va_item['settings']['applyRegularExpressions'] as $vn_regex_index => $va_regex) { if (!strlen($va_regex['match'])) { continue; } $vm_val = preg_replace("!" . str_replace("!", "\\!", $va_regex['match']) . "!" . (isset($va_regex['caseSensitive']) && (bool) $va_regex['caseSensitive'] ? '' : 'i'), $va_regex['replaceWith'], $vm_val); } } } $va_vals[$vn_i] = $vm_val; if ($o_reader->valuesCanRepeat()) { $va_row[$va_item['source']][$vn_i] = $va_row[mb_strtolower($va_item['source'])][$vn_i] = $vm_val; } else { $va_row[$va_item['source']] = $va_row[mb_strtolower($va_item['source'])] = $vm_val; } } // Process each value $vn_c = -1; foreach ($va_vals as $vn_i => $vm_val) { $vn_c++; if (isset($va_item['settings']['convertNewlinesToHTML']) && (bool) $va_item['settings']['convertNewlinesToHTML'] && is_string($vm_val)) { $vm_val = nl2br($vm_val); } // Get location in content tree for addition of new content $va_item_dest = explode(".", $va_item['destination']); $vs_item_terminal = $va_item_dest[sizeof($va_item_dest) - 1]; if (isset($va_item['settings']['restrictToTypes']) && is_array($va_item['settings']['restrictToTypes']) && !in_array($vs_type, $va_item['settings']['restrictToTypes'])) { $o_log->logInfo(_t('[%1] Skipped row %2 because of type restriction', $vs_idno, $vn_row)); continue 4; } if (isset($va_item['settings']['skipRowIfEmpty']) && (bool) $va_item['settings']['skipRowIfEmpty'] && !strlen($vm_val)) { $o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 is empty', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id)); continue 4; } if (isset($va_item['settings']['skipRowIfValue']) && is_array($va_item['settings']['skipRowIfValue']) && strlen($vm_val) && in_array($vm_val, $va_item['settings']['skipRowIfValue'])) { $o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 matches value %5', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id)); continue 4; } if (isset($va_item['settings']['skipRowIfNotValue']) && is_array($va_item['settings']['skipRowIfNotValue']) && strlen($vm_val) && !in_array($vm_val, $va_item['settings']['skipRowIfNotValue'])) { $o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 is not in list of values', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id, $vm_val)); continue 4; } if (isset($va_item['settings']['skipGroupIfEmpty']) && (bool) $va_item['settings']['skipGroupIfEmpty'] && !strlen($vm_val)) { $o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 is empty', $vs_idno, $vn_group_id, $vs_item_terminal)); continue 3; } if (isset($va_item['settings']['skipGroupIfExpression']) && strlen(trim($va_item['settings']['skipGroupIfExpression']))) { if ($vm_ret = ExpressionParser::evaluate($va_item['settings']['skipGroupIfExpression'], $va_row)) { $o_log->logInfo(_t('[%1] Skipped group %2 because expression %3 is true', $vs_idno, $vn_group_id, $va_item['settings']['skipGroupIfExpression'])); continue 3; } } if (isset($va_item['settings']['skipGroupIfValue']) && is_array($va_item['settings']['skipGroupIfValue']) && strlen($vm_val) && in_array($vm_val, $va_item['settings']['skipGroupIfValue'])) { $o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 matches value %4', $vs_idno, $vn_group_id, $vs_item_terminal, $vm_val)); continue 3; } if (isset($va_item['settings']['skipGroupIfNotValue']) && is_array($va_item['settings']['skipGroupIfNotValue']) && strlen($vm_val) && !in_array($vm_val, $va_item['settings']['skipGroupIfNotValue'])) { $o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 matches is not in list of values', $vs_idno, $vn_group_id, $vs_item_terminal)); continue 3; } if (isset($va_item['settings']['skipIfExpression']) && strlen(trim($va_item['settings']['skipIfExpression']))) { if ($vm_ret = ExpressionParser::evaluate($va_item['settings']['skipIfExpression'], $va_row)) { $o_log->logInfo(_t('[%1] Skipped mapping because expression %2 is true', $vs_idno, $va_item['settings']['skipIfExpression'])); continue 2; } } if (isset($va_item['settings']['skipIfEmpty']) && (bool) $va_item['settings']['skipIfEmpty'] && !strlen($vm_val)) { $o_log->logInfo(_t('[%1] Skipped mapping because value for %2 is empty', $vs_idno, $vs_item_terminal)); continue 2; } if ($vn_type_id_mapping_item_id && $vn_item_id == $vn_type_id_mapping_item_id) { continue; } if ($vn_idno_mapping_item_id && $vn_item_id == $vn_idno_mapping_item_id) { continue; } if (is_null($vm_val)) { continue; } // Get mapping error policy $vb_item_error_policy_is_default = false; if (!isset($va_item['settings']['errorPolicy']) || !in_array($vs_item_error_policy = $va_item['settings']['errorPolicy'], array('ignore', 'stop'))) { $vs_item_error_policy = 'ignore'; $vb_item_error_policy_is_default = true; } // if (isset($va_item['settings']['relationshipType']) && strlen($vs_rel_type = $va_item['settings']['relationshipType']) && $vs_target_table != $vs_subject_table) { $va_group_buf[$vn_c]['_relationship_type'] = $vs_rel_type; } // Is it a constant value? if (preg_match("!^_CONSTANT_:[\\d]+:(.*)!", $va_item['source'], $va_matches)) { $va_group_buf[$vn_c][$vs_item_terminal] = $va_matches[1]; // Set it and go onto the next item if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) { $va_mandatory_field_values[$vs_k] = $vm_val; } continue; } // Perform refinery call (if required) per value if (isset($va_item['settings']['refineries']) && is_array($va_item['settings']['refineries'])) { foreach ($va_item['settings']['refineries'] as $vs_refinery) { if (!$vs_refinery) { continue; } if ($o_refinery = RefineryManager::getRefineryInstance($vs_refinery)) { $va_refined_values = $o_refinery->refine($va_content_tree, $va_group, $va_item, $va_row, array('mapping' => $t_mapping, 'source' => $ps_source, 'subject' => $t_subject, 'locale_id' => $vn_locale_id, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'reader' => $o_reader, 'valueIndex' => $vn_i)); if (!$va_refined_values || is_array($va_refined_values) && !sizeof($va_refined_values)) { continue 2; } if ($o_refinery->returnsMultipleValues()) { foreach ($va_refined_values as $va_refined_value) { $va_refined_value['_errorPolicy'] = $vs_item_error_policy; if (!is_array($va_group_buf[$vn_c])) { $va_group_buf[$vn_c] = array(); } $va_group_buf[$vn_c] = array_merge($va_group_buf[$vn_c], $va_refined_value); $vn_c++; } } else { $va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy; $va_group_buf[$vn_c][$vs_item_terminal] = $va_refined_values; $vn_c++; } if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) { $va_mandatory_field_values[$vs_k] = $vm_val; } continue 2; } else { ca_data_importers::logImportError(_t('[%1] Invalid refinery %2 specified', $vs_idno, $vs_refinery)); } } } if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) { $va_mandatory_field_values[$vs_k] = $vm_val; } $vn_max_length = !is_array($vm_val) && isset($va_item['settings']['maxLength']) && (int) $va_item['settings']['maxLength'] ? (int) $va_item['settings']['maxLength'] : null; if (isset($va_item['settings']['delimiter']) && $va_item['settings']['delimiter']) { if (!is_array($va_item['settings']['delimiter'])) { $va_item['settings']['delimiter'] = array($va_item['settings']['delimiter']); } if (sizeof($va_item['settings']['delimiter'])) { foreach ($va_item['settings']['delimiter'] as $vn_index => $vs_delim) { $va_item['settings']['delimiter'][$vn_index] = preg_quote($vs_delim, "!"); } $va_val_list = preg_split("!(" . join("|", $va_item['settings']['delimiter']) . ")!", $vm_val); // Add delimited values foreach ($va_val_list as $vs_list_val) { $vs_list_val = trim(ca_data_importers::replaceValue($vs_list_val, $va_item)); if ($vn_max_length && mb_strlen($vs_list_val) > $vn_max_length) { $vs_list_val = mb_substr($vs_list_val, 0, $vn_max_length); } $va_group_buf[$vn_c] = array($vs_item_terminal => $vs_list_val, '_errorPolicy' => $vs_item_error_policy); $vn_c++; } $vn_row++; continue; // Don't add "regular" value below } } if ($vn_max_length && mb_strlen($vm_val) > $vn_max_length) { $vm_val = mb_substr($vm_val, 0, $vn_max_length); } switch ($vs_item_terminal) { case 'preferred_labels': case 'nonpreferred_labels': if ($t_instance = $o_dm->getInstanceByTableName($vs_target_table, true)) { $va_group_buf[$vn_c][$t_instance->getLabelDisplayField()] = $vm_val; } if ($o_trans) { $t_instance->setTransaction($o_trans); } if (!$vb_item_error_policy_is_default || !isset($va_group_buf[$vn_c]['_errorPolicy'])) { if (is_array($va_group_buf[$vn_c])) { $va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy; } } if ($vs_item_terminal == 'preferred_labels') { $vs_preferred_label_for_log = $vm_val; } break; default: $va_group_buf[$vn_c][$vs_item_terminal] = $vm_val; if (!$vb_item_error_policy_is_default || !isset($va_group_buf[$vn_c]['_errorPolicy'])) { if (is_array($va_group_buf[$vn_c])) { $va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy; } } break; } } // end foreach($va_vals as $vm_val) } foreach ($va_group_buf as $vn_group_index => $va_group_data) { $va_ptr =& $va_content_tree; foreach ($va_group_tmp as $vs_tmp) { if (!is_array($va_ptr[$vs_tmp])) { $va_ptr[$vs_tmp] = array(); } $va_ptr =& $va_ptr[$vs_tmp]; if ($vs_tmp == $vs_target_table) { // add numeric index after table to ensure repeat values don't overwrite each other $va_parent =& $va_ptr; $va_ptr[] = array(); $va_ptr =& $va_ptr[sizeof($va_ptr) - 1]; } } $va_ptr = $va_group_data; } } // // Process out self-relationships // if (is_array($va_content_tree[$vs_subject_table])) { $va_self_related_content = array(); foreach ($va_content_tree[$vs_subject_table] as $vn_i => $va_element_data) { if (isset($va_element_data['_relationship_type'])) { $va_self_related_content[] = $va_element_data; unset($va_content_tree[$vs_subject_table][$vn_i]); } } if (sizeof($va_self_related_content) > 0) { $va_content_tree["related.{$vs_subject_table}"] = $va_self_related_content; } } $vn_row++; $o_log->logDebug(_t('Finished building content tree for %1 at %2 seconds', $vs_idno, $t->getTime(4))); $o_log->logDebug(_t("Content tree is\n%1", print_R($va_content_tree, true))); // // Process data in subject record // //print_r($va_content_tree); //die("END\n\n"); //continue; $opa_app_plugin_manager->hookDataImportContentTree(array('mapping' => $t_mapping, 'content_tree' => &$va_content_tree, 'idno' => &$vs_idno, 'transaction' => &$o_trans, 'log' => &$o_log, 'reader' => $o_reader, 'environment' => $va_environment, 'importEvent' => $o_event, 'importEventSource' => $vn_row)); //print_r($va_content_tree); //die("done\n"); if (!sizeof($va_content_tree) && !str_replace("%", "", $vs_idno)) { continue; } if (!$t_subject->getPrimaryKey()) { $o_event->beginItem($vn_row, $t_subject->tableNum(), 'I'); $t_subject->setMode(ACCESS_WRITE); $t_subject->set($vs_type_id_fld, $vs_type); if ($vb_idno_is_template) { $t_subject->setIdnoWithTemplate($vs_idno); } else { $t_subject->set($vs_idno_fld, $vs_idno, array('assumeIdnoForRepresentationID' => true, 'assumeIdnoStubForLotID' => true)); // assumeIdnoStubForLotID forces ca_objects.lot_id values to always be considered as a potential idno_stub first, before use as a ca_objects.lot_id } // Look for parent_id in the content tree $vs_parent_id_fld = $t_subject->getProperty('HIERARCHY_PARENT_ID_FLD'); foreach ($va_content_tree as $vs_table_name => $va_content) { if ($vs_table_name == $vs_subject_table) { foreach ($va_content as $va_element_data) { foreach ($va_element_data as $vs_element => $va_element_content) { switch ($vs_element) { case $vs_parent_id_fld: if ($va_element_content[$vs_parent_id_fld]) { $t_subject->set($vs_parent_id_fld, $va_element_content[$vs_parent_id_fld], array('treatParentIDAsIdno' => true)); } break; } } } } } foreach ($va_mandatory_field_mapping_ids as $vs_mandatory_field => $vn_mandatory_mapping_item_id) { $t_subject->set($vs_mandatory_field, $va_mandatory_field_values[$vs_mandatory_field], array('assumeIdnoStubForLotID' => true)); } $t_subject->insert(); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not insert new record"), array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_import_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to import error policy')); if ($vb_use_ncurses) { ncurses_end(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); if ($o_trans) { $o_trans->rollback(); } return false; } continue; } $o_log->logDebug(_t('Created idno %1 at %2 seconds', $vs_idno, $t->getTime(4))); } else { $o_event->beginItem($vn_row, $t_subject->tableNum(), 'U'); // update $t_subject->setMode(ACCESS_WRITE); if ($vb_idno_is_template) { $t_subject->setIdnoWithTemplate($vs_idno); } else { $t_subject->set($vs_idno_fld, $vs_idno, array('assumeIdnoStubForLotID' => true)); // assumeIdnoStubForLotID forces ca_objects.lot_id values to always be considered as a potential idno_stub first, before use as a ca_objects.lot_di } $t_subject->update(); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not update matched record"), array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_import_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to import error policy')); if ($vb_use_ncurses) { ncurses_end(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); if ($o_trans) { $o_trans->rollback(); } return false; } continue; } $t_subject->clearErrors(); if (sizeof($va_preferred_label_mapping_ids) && $t_subject->getPreferredLabelCount() > 0) { $t_subject->removeAllLabels(__CA_LABEL_TYPE_PREFERRED__); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not update remove preferred labels from matched record"), array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_import_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to import error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); return false; } } } $o_log->logDebug(_t('Updated idno %1 at %2 seconds', $vs_idno, $t->getTime(4))); } if ($vs_idno_fld && ($o_idno = $t_subject->getIDNoPlugInInstance())) { $va_values = $o_idno->htmlFormValuesAsArray($vs_idno_fld, $t_subject->get($vs_idno_fld)); if (!is_array($va_values)) { $va_values = array($va_values); } if (($vs_proc_idno = join($o_idno->getSeparator(), $va_values)) && $vs_proc_idno != $vs_idno) { $t_subject->set($vs_idno_fld, $vs_proc_idno); $t_subject->update(); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could update idno"), array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_import_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to import error policy')); if ($vb_use_ncurses) { ncurses_end(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); if ($o_trans) { $o_trans->rollback(); } return false; } continue; } } } $va_elements_set_for_this_record = array(); foreach ($va_content_tree as $vs_table_name => $va_content) { if ($vs_table_name == $vs_subject_table) { foreach ($va_content as $vn_i => $va_element_data) { foreach ($va_element_data as $vs_element => $va_element_content) { if (is_array($va_element_content)) { $vs_item_error_policy = $va_element_content['_errorPolicy']; unset($va_element_content['_errorPolicy']); } else { $vs_item_error_policy = null; } $t_subject->clearErrors(); $t_subject->setMode(ACCESS_WRITE); switch ($vs_element) { case 'preferred_labels': $t_subject->addLabel($va_element_content, $vn_locale_id, isset($va_element_content['type_id']) ? $va_element_content['type_id'] : null, true); if ($t_subject->numErrors() == 0) { $vb_output_subject_preferred_label = true; } if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add preferred label to %2. Record was deleted because no preferred label could be applied: ", $vs_idno, $t_subject->tableName()), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); $t_subject->delete(true, array('hard' => false)); if ($vs_import_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to import error policy %1', $vs_import_error_policy)); if ($vb_use_ncurses) { ncurses_end(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); if ($o_trans) { $o_trans->rollback(); } return false; } if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } continue 5; } break; case 'nonpreferred_labels': $t_subject->addLabel($va_element_content, $vn_locale_id, isset($va_element_content['type_id']) ? $va_element_content['type_id'] : null, false); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add non-preferred label to %2:", $vs_idno, $t_subject->tableName()), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); if ($o_trans) { $o_trans->rollback(); } return false; } } break; default: if ($t_subject->hasField($vs_element)) { $t_subject->set($vs_element, $va_element_content[$vs_element], array('assumeIdnoStubForLotID' => true)); $t_subject->update(); break; } if ($vs_subject_table == 'ca_representation_annotations' && $vs_element == 'properties') { foreach ($va_element_content as $vs_prop => $vs_prop_val) { $t_subject->setPropertyValue($vs_prop, $vs_prop_val); } break; } if (is_array($va_element_content)) { $va_element_content['locale_id'] = $vn_locale_id; } if (!isset($va_elements_set_for_this_record[$vs_element]) && !$va_elements_set_for_this_record[$vs_element] && in_array($vs_existing_record_policy, array('merge_on_idno_with_replace', 'merge_on_preferred_labels_with_replace', 'merge_on_idno_and_preferred_labels_with_replace'))) { $t_subject->removeAttributes($vs_element, array('force' => true)); } $va_elements_set_for_this_record[$vs_element] = true; $t_subject->addAttribute($va_element_content, $vs_element, null, array('showRepeatCountErrors' => true, 'alwaysTreatValueAsIdno' => true)); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Failed to add value for %2; values were %3: ", $vs_idno, $vs_element, ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); if ($o_trans) { $o_trans->rollback(); } return false; } } $t_subject->update(); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Invalid %2; values were %3: ", $vs_idno, $vs_element, ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); if ($o_trans) { $o_trans->rollback(); } return false; } } break; } } } } else { // related $vs_table_name = preg_replace('!^related\\.!', '', $vs_table_name); foreach ($va_content as $vn_i => $va_element_data) { $va_match_on = caGetOption('_matchOn', $va_element_data, null); $vb_dont_create = caGetOption('_dontCreate', $va_element_data, null); $va_data_for_rel_table = $va_element_data; $va_nonpreferred_labels = isset($va_data_for_rel_table['nonpreferred_labels']) ? $va_data_for_rel_table['nonpreferred_labels'] : null; unset($va_data_for_rel_table['preferred_labels']); unset($va_data_for_rel_table['_relationship_type']); unset($va_data_for_rel_table['_type']); unset($va_data_for_rel_table['_parent_id']); unset($va_data_for_rel_table['_errorPolicy']); unset($va_data_for_rel_table['_matchOn']); $va_data_for_rel_table = array_merge($va_data_for_rel_table, ca_data_importers::_extractIntrinsicValues($va_data_for_rel_table, $vs_table_name)); $t_subject->clearErrors(); switch ($vs_table_name) { case 'ca_objects': if ($vn_rel_id = DataMigrationUtils::getObjectID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related object with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_object_lots': $vs_idno_stub = null; if (is_array($va_element_data['idno_stub'])) { $vs_idno_stub = isset($va_element_data['idno_stub']['idno_stub']) ? $va_element_data['idno_stub']['idno_stub'] : ''; } else { $vs_idno_stub = isset($va_element_data['idno_stub']) ? $va_element_data['idno_stub'] : ''; } if ($vn_rel_id = DataMigrationUtils::getObjectLotID($vs_idno_stub, $va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno_stub']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related object lot with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_entities': if ($vn_rel_id = DataMigrationUtils::getEntityID($va_element_data['preferred_labels'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related entity with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_places': if ($vn_rel_id = DataMigrationUtils::getPlaceID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related place with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_collections': if ($vn_rel_id = DataMigrationUtils::getCollectionID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, $vs_rel_type, null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related collection with relationship %2:", $vs_idno, $vs_rel_type), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_occurrences': if ($vn_rel_id = DataMigrationUtils::getOccurrenceID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, $vs_rel_type, null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related occurrence with relationship %2:", $vs_idno, $vs_rel_type), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_storage_locations': if ($vn_rel_id = DataMigrationUtils::getStorageLocationID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related storage location with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_list_items': $va_data_for_rel_table['is_enabled'] = 1; $va_data_for_rel_table['preferred_labels'] = $va_element_data['preferred_labels']; if ($vn_rel_id = DataMigrationUtils::getListItemID($va_element_data['_list'], $va_element_data['idno'] ? $va_element_data['idno'] : null, $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related list item with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_object_representations': if ($vn_rel_id = DataMigrationUtils::getObjectRepresentationID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels, 'matchMediaFilesWithoutExtension' => true))) { $t_subject->linkRepresentation($vn_rel_id, null, null, null, null, array('type_id' => trim($va_element_data['_relationship_type']), 'is_primary' => true)); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related object representation with:", $vs_idno), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } // // if (($vs_subject_table_name == 'ca_objects') && $va_element_data['media']['media']) { // unset($va_data_for_rel_table['media']); // // foreach($va_data_for_rel_table as $vs_key => $vm_val) { // // Attributes, including intrinsics are in two-level format, eg. idno is $va_attributes['idno']['idno'] // // but addRepresentations() expects intrinsics to be single level (eg. $va_attributes['idno']) so // // we do some rewriting here // if (is_array($vm_val) && isset($vm_val[$vs_key])) { // $va_data_for_rel_table[$vs_key] = $vm_val[$vs_key]; // } // } // // if (!($t_subject->addRepresentation($va_element_data['media']['media'], isset($va_element_data['_type']) ? $va_element_data['_type'] : caGetDefaultItemID('object_representation_types'), $vn_locale_id, 0, 0, true, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on)))) { // $vs_error = join("; ", $t_subject->getErrors()); // ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); // if ($vs_item_error_policy == 'stop') { // $o_log->logAlert(_t('Import stopped due to mapping error policy')); // if($vb_use_ncurses) { ncurses_end(); } // if ($o_trans) { $o_trans->rollback(); } // return false; // } // } // } break; case 'ca_loans': if ($vn_rel_id = DataMigrationUtils::getLoanID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related loan with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; case 'ca_movements': if ($vn_rel_id = DataMigrationUtils::getMovementID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) { if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) { break; } $t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial'])); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related movement with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_item_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to mapping error policy')); if ($vb_use_ncurses) { ncurses_end(); } if ($o_trans) { $o_trans->rollback(); } return false; } } } break; } if (is_array($va_element_data['_related_related']) && sizeof($va_element_data['_related_related'])) { foreach ($va_element_data['_related_related'] as $vs_rel_rel_table => $va_rel_rels) { foreach ($va_rel_rels as $vn_i => $va_rel_rel) { if (!($t_rel_instance = $o_dm->getInstanceByTableName($vs_table_name))) { $o_log->logWarn(_t("[%1] Could not instantiate related table %2", $vs_idno, $vs_table_name)); continue; } if ($o_trans) { $t_rel_instance->setTransaction($o_trans); } if ($t_rel_instance->load($vn_rel_id)) { if ($t_rel_rel = $t_rel_instance->addRelationship($vs_rel_rel_table, $va_rel_rel['id'], $va_rel_rel['_relationship_type'])) { $o_log->logInfo(_t('[%1] Related %2 (%3) to related %4 with relationship %5', $vs_idno, $o_dm->getTableProperty($vs_rel_rel_table, 'NAME_SINGULAR'), $va_rel_rel['id'], $t_rel_instance->getProperty('NAME_SINGULAR'), trim($va_rel_rel['_relationship_type']))); } else { if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related %2 (%3) to related %4 with relationship %5:", $vs_idno, $o_dm->getTableProperty($vs_rel_rel_table, 'NAME_SINGULAR'), $va_rel_rel['id'], $t_rel_instance->getProperty('NAME_SINGULAR'), trim($va_rel_rel['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); } } } } } } } } } // $t_subject->update(); // // if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Invalid %2; values were %3: ", $vs_idno, 'attributes', ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { // ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); // if ($vs_item_error_policy == 'stop') { // $o_log->logAlert(_t('Import stopped due to mapping error policy')); // if($vb_use_ncurses) { ncurses_end(); } // // $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); // // if ($o_trans) { $o_trans->rollback(); } // return false; // } // } // $o_log->logDebug(_t('Finished inserting content tree for %1 at %2 seconds into database', $vs_idno, $t->getTime(4))); if (!$vb_output_subject_preferred_label && $t_subject->getPreferredLabelCount() == 0) { $t_subject->addLabel(array($vs_label_display_fld => '???'), $vn_locale_id, null, true); if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add default label", $vs_idno), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) { ca_data_importers::logImportError($vs_error, $va_log_import_error_opts); if ($vs_import_error_policy == 'stop') { $o_log->logAlert(_t('Import stopped due to import error policy')); if ($vb_use_ncurses) { ncurses_end(); } $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno)); if ($o_trans) { $o_trans->rollback(); } return false; } } } $o_log->logInfo(_t('[%1] Imported %2 as %3 ', $vs_idno, $vs_preferred_label_for_log, $vs_subject_table_name)); $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_SUCCESS__, _t('Imported %1', $vs_idno)); ca_data_importers::$s_num_records_processed++; } $o_log->logInfo(_t('Import of %1 completed using mapping %2: %3 imported/%4 skipped/%5 errors', $ps_source, $t_mapping->get('importer_code'), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_records_skipped, ca_data_importers::$s_num_import_errors)); //if ($vb_show_cli_progress_bar) { $o_progress->finish(); //} if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) { $ps_callback($po_request, $pn_file_number, $pn_number_of_files, $ps_source, $vn_num_items, $vn_num_items, _t('Import completed'), time() - $vn_start_time, memory_get_usage(true), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_import_errors); } if (isset($pa_options['reportCallback']) && ($ps_callback = $pa_options['reportCallback'])) { $va_general = array('elapsedTime' => time() - $vn_start_time, 'numErrors' => ca_data_importers::$s_num_import_errors, 'numProcessed' => ca_data_importers::$s_num_records_processed); $ps_callback($po_request, $va_general, ca_data_importers::$s_import_error_list, true); } if ($vb_use_ncurses) { ncurses_end(); } if ($pb_dry_run) { if ($o_trans) { $o_trans->rollback(); } $o_log->logInfo(_t('Rollback successful import run in "dry run" mode')); } else { if ($o_trans) { $o_trans->commit(); } } return true; }
// Check Request for IPN if ($_GET['q'] == 'ipn') { require_once "Handler/PaypalHandler.php"; // PayPal has its own logger $paypal_log = new KLogger($_SERVER['DOCUMENT_ROOT'] . '/' . $CONFIG['paths']['base_url'] . '/logs/paypal', Klogger::DEBUG); $paypal_log->logInfo('Received IPN Request'); $handler = new PaypalHandler(); $status = $handler->process(); // success could have multiple payments to log if ($status['status'] == true) { foreach ($status['message'] as $payment) { $paypal_log->logInfo('Registration #' . $payment->order_id . ' - ' . $payment->payment_amount . ' paid in full'); } } else { $paypal_log->logError($status['message']); $paypal_log->logDebug(print_r(debug_backtrace(), true)); } // no more IPN processing required exit; } // Set some template defaults global $smarty; $smarty->assign('app_name', variable_get('app_name', 'Leaguerunner')); $smarty->assign('app_admin_name', variable_get('app_admin_name', 'Leaguerunner Admin')); $smarty->assign('app_admin_email', variable_get('app_admin_email', 'webmaster@localhost')); $smarty->assign('app_version', '2.8.5'); $smarty->assign('base_url', $CONFIG['paths']['base_url']); $smarty->assign('site_name', 'Your Ultimate Club'); $smarty->assign('site_slogan', 'Slogan about Your Ultimate Club'); //TODO NOTE: Testing Only, MUST BE REMOVED for Production //$smarty->force_compile = true;
// Include our configuration (holding defines for the requires) require_once BASEPATH . 'include/config/global.inc.php'; // We include all needed files here, even though our templates could load them themself require_once INCLUDE_DIR . '/autoloader.inc.php'; // Command line switches array_shift($argv); foreach ($argv as $option) { switch ($option) { case '-f': $monitoring->setStatus($cron_name . "_disabled", "yesno", 0); break; } } // Load 3rd party logging library for running crons $log = new KLogger('logs/' . $cron_name . '.txt', KLogger::INFO); $log->LogDebug('Starting ' . $cron_name); // Load the start time for later runtime calculations for monitoring $cron_start[$cron_name] = microtime(true); // Check if our cron is activated if ($monitoring->isDisabled($cron_name)) { $log->logFatal('Cronjob is currently disabled due to errors, use -f option to force running cron.'); $monitoring->endCronjob($cron_name, 'E0018', 1, true, false); } // Mark cron as running for monitoring $log->logDebug('Marking cronjob as running for monitoring'); $monitoring->setStatus($cron_name . '_starttime', 'date', time()); // Check if we need to halt our crons due to an outstanding upgrade if ($setting->getValue('DB_VERSION') != DB_VERSION || $config['version'] != CONFIG_VERSION) { $log->logFatal('Cronjob is currently disabled due to required upgrades. Import any outstanding SQL files and check your configuration file.'); $monitoring->endCronjob($cron_name, 'E0075', 0, true, false); }
/** * @param RequestHTTP $po_request * @param null|array $pa_options * progressCallback = * reportCallback = * sendMail = * log = log directory path * logLevel = KLogger constant for minimum log level to record. Default is KLogger::INFO. Constants are, in descending order of shrillness: * KLogger::EMERG = Emergency messages (system is unusable) * KLogger::ALERT = Alert messages (action must be taken immediately) * KLogger::CRIT = Critical conditions * KLogger::ERR = Error conditions * KLogger::WARN = Warnings * KLogger::NOTICE = Notices (normal but significant conditions) * KLogger::INFO = Informational messages * KLogger::DEBUG = Debugging messages * @return array */ public static function importMediaFromDirectory($po_request, $pa_options = null) { global $g_ui_locale_id; $vs_log_dir = caGetOption('log', $pa_options, __CA_APP_DIR__ . "/log"); $vs_log_level = caGetOption('logLevel', $pa_options, "INFO"); if (!is_writeable($vs_log_dir)) { $vs_log_dir = caGetTempDirPath(); } $vn_log_level = BatchProcessor::_logLevelStringToNumber($vs_log_level); $o_log = new KLogger($vs_log_dir, $vn_log_level); $vs_import_target = caGetOption('importTarget', $pa_options, 'ca_objects'); $t_instance = $po_request->getAppDatamodel()->getInstance($vs_import_target); $o_eventlog = new Eventlog(); $t_set = new ca_sets(); $va_notices = $va_errors = array(); $vb_we_set_transaction = false; $o_trans = isset($pa_options['transaction']) && $pa_options['transaction'] ? $pa_options['transaction'] : null; if (!$o_trans) { $vb_we_set_transaction = true; $o_trans = new Transaction($t_set->getDb()); } $o_batch_log = new Batchlog(array('user_id' => $po_request->getUserID(), 'batch_type' => 'MI', 'table_num' => (int) $t_instance->tableNum(), 'notes' => '', 'transaction' => $o_trans)); if (!is_dir($pa_options['importFromDirectory'])) { $o_eventlog->log(array("CODE" => 'ERR', "SOURCE" => "mediaImport", "MESSAGE" => $vs_msg = _t("Specified import directory '%1' is invalid", $pa_options['importFromDirectory']))); $o_log->logError($vs_msg); return null; } $vs_batch_media_import_root_directory = $po_request->config->get('batch_media_import_root_directory'); if (!preg_match("!^{$vs_batch_media_import_root_directory}!", $pa_options['importFromDirectory'])) { $o_eventlog->log(array("CODE" => 'ERR', "SOURCE" => "mediaImport", "MESSAGE" => $vs_msg = _t("Specified import directory '%1' is invalid", $pa_options['importFromDirectory']))); $o_log->logError($vs_msg); return null; } if (preg_match("!\\.\\./!", $pa_options['importFromDirectory'])) { $o_eventlog->log(array("CODE" => 'ERR', "SOURCE" => "mediaImport", "MESSAGE" => $vs_msg = _t("Specified import directory '%1' is invalid", $pa_options['importFromDirectory']))); $o_log->logError($vs_msg); return null; } $vb_include_subdirectories = (bool) $pa_options['includeSubDirectories']; $vb_delete_media_on_import = (bool) $pa_options['deleteMediaOnImport']; $vs_import_mode = $pa_options['importMode']; $vs_match_mode = $pa_options['matchMode']; $vn_type_id = $pa_options[$vs_import_target . '_type_id']; $vn_rep_type_id = $pa_options['ca_object_representations_type_id']; $va_limit_matching_to_type_ids = $pa_options[$vs_import_target . '_limit_matching_to_type_ids']; $vn_access = $pa_options[$vs_import_target . '_access']; $vn_object_representation_access = $pa_options['ca_object_representations_access']; $vn_status = $pa_options[$vs_import_target . '_status']; $vn_object_representation_status = $pa_options['ca_object_representations_status']; $vn_rel_type_id = isset($pa_options[$vs_import_target . '_representation_relationship_type']) ? $pa_options[$vs_import_target . '_representation_relationship_type'] : null; $vn_mapping_id = $pa_options[$vs_import_target . '_mapping_id']; $vn_object_representation_mapping_id = $pa_options['ca_object_representations_mapping_id']; $vs_idno_mode = $pa_options['idnoMode']; $vs_idno = $pa_options['idno']; $vs_representation_idno_mode = $pa_options['representationIdnoMode']; $vs_representation_idno = $pa_options['representation_idno']; $vs_set_mode = $pa_options['setMode']; $vs_set_create_name = $pa_options['setCreateName']; $vn_set_id = $pa_options['set_id']; $vn_locale_id = $pa_options['locale_id']; $vs_skip_file_list = $pa_options['skipFileList']; $vs_skip_file_list = $pa_options['skipFileList']; $vb_allow_duplicate_media = $pa_options['allowDuplicateMedia']; $va_relationship_type_id_for = array(); if (is_array($va_create_relationship_for = $pa_options['create_relationship_for'])) { foreach ($va_create_relationship_for as $vs_rel_table) { $va_relationship_type_id_for[$vs_rel_table] = $pa_options['relationship_type_id_for_' . $vs_rel_table]; } } if (!$vn_locale_id) { $vn_locale_id = $g_ui_locale_id; } $va_files_to_process = caGetDirectoryContentsAsList($pa_options['importFromDirectory'], $vb_include_subdirectories); $o_log->logInfo(_t('Found %1 files in directory \'%2\'', sizeof($va_files_to_process), $pa_options['importFromDirectory'])); if ($vs_set_mode == 'add') { $t_set->load($vn_set_id); } else { if ($vs_set_mode == 'create' && $vs_set_create_name) { $va_set_ids = $t_set->getSets(array('user_id' => $po_request->getUserID(), 'table' => $t_instance->tableName(), 'access' => __CA_SET_EDIT_ACCESS__, 'setIDsOnly' => true, 'name' => $vs_set_create_name)); $vn_set_id = null; if (is_array($va_set_ids) && sizeof($va_set_ids) > 0) { $vn_possible_set_id = array_shift($va_set_ids); if ($t_set->load($vn_possible_set_id)) { $vn_set_id = $t_set->getPrimaryKey(); } } else { $vs_set_code = mb_substr(preg_replace("![^A-Za-z0-9_\\-]+!", "_", $vs_set_create_name), 0, 100); if ($t_set->load(array('set_code' => $vs_set_code))) { $vn_set_id = $t_set->getPrimaryKey(); } } if (!$t_set->getPrimaryKey()) { $t_set->setMode(ACCESS_WRITE); $t_set->set('user_id', $po_request->getUserID()); $t_set->set('type_id', $po_request->config->get('ca_sets_default_type')); $t_set->set('table_num', $t_instance->tableNum()); $t_set->set('set_code', $vs_set_code); $t_set->insert(); if ($t_set->numErrors()) { $va_notices['create_set'] = array('idno' => '', 'label' => _t('Create set %1', $vs_set_create_name), 'message' => $vs_msg = _t('Failed to create set %1: %2', $vs_set_create_name, join("; ", $t_set->getErrors())), 'status' => 'SET ERROR'); $o_log->logError($vs_msg); } else { $t_set->addLabel(array('name' => $vs_set_create_name), $vn_locale_id, null, true); if ($t_set->numErrors()) { $va_notices['add_set_label'] = array('idno' => '', 'label' => _t('Add label to set %1', $vs_set_create_name), 'message' => $vs_msg = _t('Failed to add label to set: %1', join("; ", $t_set->getErrors())), 'status' => 'SET ERROR'); $o_log->logError($vs_msg); } $vn_set_id = $t_set->getPrimaryKey(); } } } else { $vn_set_id = null; // no set } } if ($t_set->getPrimaryKey() && !$t_set->haveAccessToSet($po_request->getUserID(), __CA_SET_EDIT_ACCESS__)) { $va_notices['set_access'] = array('idno' => '', 'label' => _t('You do not have access to set %1', $vs_set_create_name), 'message' => $vs_msg = _t('Cannot add to set %1 because you do not have edit access', $vs_set_create_name), 'status' => 'SET ERROR'); $o_log->logError($vs_msg); $vn_set_id = null; $t_set = new ca_sets(); } $vn_num_items = sizeof($va_files_to_process); // Get list of regex packages that user can use to extract object idno's from filenames $va_regex_list = caBatchGetMediaFilenameToIdnoRegexList(array('log' => $o_log)); // Get list of replacements that user can use to transform file names to match object idnos $va_replacements_list = caBatchGetMediaFilenameReplacementRegexList(array('log' => $o_log)); // Get list of files (or file name patterns) to skip $va_skip_list = preg_split("![\r\n]+!", $vs_skip_file_list); foreach ($va_skip_list as $vn_i => $vs_skip) { if (!strlen($va_skip_list[$vn_i] = trim($vs_skip))) { unset($va_skip_list[$vn_i]); } } $vn_c = 0; $vn_start_time = time(); $va_report = array(); foreach ($va_files_to_process as $vs_file) { $va_tmp = explode("/", $vs_file); $f = array_pop($va_tmp); $d = array_pop($va_tmp); array_push($va_tmp, $d); $vs_directory = join("/", $va_tmp); // Skip file names using $vs_skip_file_list if (BatchProcessor::_skipFile($f, $va_skip_list)) { $o_log->logInfo(_t('Skipped file %1 because it was on the skipped files list', $f)); continue; } $vs_relative_directory = preg_replace("!{$vs_batch_media_import_root_directory}[/]*!", "", $vs_directory); // does representation already exist? if (!$vb_allow_duplicate_media && ($t_dupe = ca_object_representations::mediaExists($vs_file))) { $va_notices[$vs_relative_directory . '/' . $f] = array('idno' => '', 'label' => $f, 'message' => $vs_msg = _t('Skipped %1 from %2 because it already exists %3', $f, $vs_relative_directory, caEditorLink($po_request, _t('(view)'), 'button', 'ca_object_representations', $t_dupe->getPrimaryKey())), 'status' => 'SKIPPED'); $o_log->logInfo($vs_msg); continue; } $t_instance = $po_request->getAppDatamodel()->getInstance($vs_import_target, false); $t_instance->setTransaction($o_trans); $vs_modified_filename = $f; $va_extracted_idnos_from_filename = array(); if (in_array($vs_import_mode, array('TRY_TO_MATCH', 'ALWAYS_MATCH')) || is_array($va_create_relationship_for) && sizeof($va_create_relationship_for)) { foreach ($va_regex_list as $vs_regex_name => $va_regex_info) { $o_log->logDebug(_t("Processing mediaFilenameToObjectIdnoRegexes entry %1", $vs_regex_name)); foreach ($va_regex_info['regexes'] as $vs_regex) { switch ($vs_match_mode) { case 'DIRECTORY_NAME': $va_names_to_match = array($d); $o_log->logDebug(_t("Trying to match on directory '%1'", $d)); break; case 'FILE_AND_DIRECTORY_NAMES': $va_names_to_match = array($f, $d); $o_log->logDebug(_t("Trying to match on directory '%1' and file name '%2'", $d, $f)); break; default: case 'FILE_NAME': $va_names_to_match = array($f); $o_log->logDebug(_t("Trying to match on file name '%1'", $f)); break; } // are there any replacements? if so, try to match each element in $va_names_to_match AND all results of the replacements if (is_array($va_replacements_list) && sizeof($va_replacements_list) > 0) { $va_names_to_match_copy = $va_names_to_match; foreach ($va_names_to_match_copy as $vs_name) { foreach ($va_replacements_list as $vs_replacement_code => $va_replacement) { if (isset($va_replacement['search']) && is_array($va_replacement['search'])) { $va_replace = caGetOption('replace', $va_replacement); $va_search = array(); foreach ($va_replacement['search'] as $vs_search) { $va_search[] = '!' . $vs_search . '!'; } $vs_replacement_result = @preg_replace($va_search, $va_replace, $vs_name); if (is_null($vs_replacement_result)) { $o_log->logError(_t("There was an error in preg_replace while processing replacement %1.", $vs_replacement_code)); } if ($vs_replacement_result && strlen($vs_replacement_result) > 0) { $o_log->logDebug(_t("The result for replacement with code %1 applied to value '%2' is '%3' and was added to the list of file names used for matching.", $vs_replacement_code, $vs_name, $vs_replacement_result)); $va_names_to_match[] = $vs_replacement_result; } } else { $o_log->logDebug(_t("Skipped replacement %1 because no search expression was defined.", $vs_replacement_code)); } } } } $o_log->logDebug("Names to match: " . print_r($va_names_to_match, true)); foreach ($va_names_to_match as $vs_match_name) { if (preg_match('!' . $vs_regex . '!', $vs_match_name, $va_matches)) { $o_log->logDebug(_t("Matched name %1 on regex %2", $vs_match_name, $vs_regex)); if (!$vs_idno || strlen($va_matches[1]) < strlen($vs_idno)) { $vs_idno = $va_matches[1]; } if (!$vs_modified_filename || strlen($vs_modified_filename) > strlen($va_matches[1])) { $vs_modified_filename = $va_matches[1]; } $va_extracted_idnos_from_filename[] = $va_matches[1]; if (in_array($vs_import_mode, array('TRY_TO_MATCH', 'ALWAYS_MATCH'))) { if (!is_array($va_fields_to_match_on = $po_request->config->getList('batch_media_import_match_on')) || !sizeof($va_fields_to_match_on)) { $batch_media_import_match_on = array('idno'); } $vs_bool = 'OR'; $va_values = array(); foreach ($va_fields_to_match_on as $vs_fld) { if (in_array($vs_fld, array('preferred_labels', 'nonpreferred_labels'))) { $va_values[$vs_fld] = array($vs_fld => array('name' => $va_matches[1])); } else { $va_values[$vs_fld] = $va_matches[1]; } } if (is_array($va_limit_matching_to_type_ids) && sizeof($va_limit_matching_to_type_ids) > 0) { $va_values['type_id'] = $va_limit_matching_to_type_ids; $vs_bool = 'AND'; } $o_log->logDebug("Trying to find records using boolean {$vs_bool} and values " . print_r($va_values, true)); if (class_exists($vs_import_target) && ($vn_id = $vs_import_target::find($va_values, array('returnAs' => 'firstId', 'boolean' => $vs_bool)))) { if ($t_instance->load($vn_id)) { $va_notices[$vs_relative_directory . '/' . $vs_match_name . '_match'] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $t_instance->getLabelForDisplay(), 'message' => $vs_msg = _t('Matched media %1 from %2 to %3 using expression "%4"', $f, $vs_relative_directory, caGetTableDisplayName($vs_import_target, false), $va_regex_info['displayName']), 'status' => 'MATCHED'); $o_log->logInfo($vs_msg); } break 3; } } } else { $o_log->logDebug(_t("Couldn't match name %1 on regex %2", $vs_match_name, $vs_regex)); } } } } } if (!$t_instance->getPrimaryKey()) { // Use filename as idno if all else fails if ($t_instance->load(array('idno' => $f, 'deleted' => 0))) { $va_notices[$vs_relative_directory . '/' . $f . '_match'] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $t_instance->getLabelForDisplay(), 'message' => $vs_msg = _t('Matched media %1 from %2 to %3 using filename', $f, $vs_relative_directory, caGetTableDisplayName($vs_import_target, false)), 'status' => 'MATCHED'); $o_log->logInfo($vs_msg); } } switch ($vs_representation_idno_mode) { case 'filename': // use the filename as identifier $vs_rep_idno = $f; break; case 'filename_no_ext': // use filename without extension as identifier $vs_rep_idno = preg_replace('/\\.[^.\\s]{3,4}$/', '', $f); break; case 'directory_and_filename': // use the directory + filename as identifier $vs_rep_idno = $d . '/' . $f; break; default: // use idno from form $vs_rep_idno = $vs_representation_idno; break; } $t_new_rep = null; if ($t_instance->getPrimaryKey() && $t_instance instanceof RepresentableBaseModel) { // found existing object $t_instance->setMode(ACCESS_WRITE); $t_new_rep = $t_instance->addRepresentation($vs_directory . '/' . $f, $vn_rep_type_id, $vn_locale_id, $vn_object_representation_status, $vn_object_representation_access, false, array('idno' => $vs_rep_idno), array('original_filename' => $f, 'returnRepresentation' => true, 'type_id' => $vn_rel_type_id)); if ($t_instance->numErrors()) { $o_eventlog->log(array("CODE" => 'ERR', "SOURCE" => "mediaImport", "MESSAGE" => _t("Error importing {$f} from {$vs_directory}: %1", join('; ', $t_instance->getErrors())))); $va_errors[$vs_relative_directory . '/' . $f] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $t_instance->getLabelForDisplay(), 'errors' => $t_instance->errors(), 'message' => $vs_msg = _t("Error importing %1 from %2: %3", $f, $vs_relative_directory, join('; ', $t_instance->getErrors())), 'status' => 'ERROR'); $o_log->logError($vs_msg); $o_trans->rollback(); continue; } else { if ($vb_delete_media_on_import) { @unlink($vs_directory . '/' . $f); } } } else { // should we create new record? if (in_array($vs_import_mode, array('TRY_TO_MATCH', 'DONT_MATCH'))) { $t_instance->setMode(ACCESS_WRITE); $t_instance->set('type_id', $vn_type_id); $t_instance->set('locale_id', $vn_locale_id); $t_instance->set('status', $vn_status); $t_instance->set('access', $vn_access); // for places, take first hierarchy we can find. in most setups there is but one. we might wanna make this configurable via setup screen at some point if ($t_instance->hasField('hierarchy_id')) { $va_hierarchies = $t_instance->getHierarchyList(); reset($va_hierarchies); $vn_hierarchy_id = key($va_hierarchies); $t_instance->set('hierarchy_id', $vn_hierarchy_id); } switch ($vs_idno_mode) { case 'filename': // use the filename as identifier $t_instance->set('idno', $f); break; case 'filename_no_ext': // use filename without extension as identifier $f_no_ext = preg_replace('/\\.[^.\\s]{3,4}$/', '', $f); $t_instance->set('idno', $f_no_ext); break; case 'directory_and_filename': // use the directory + filename as identifier $t_instance->set('idno', $d . '/' . $f); break; default: // Calculate identifier using numbering plugin $o_numbering_plugin = $t_instance->getIDNoPlugInInstance(); if (!($vs_sep = $o_numbering_plugin->getSeparator())) { $vs_sep = ''; } if (!is_array($va_idno_values = $o_numbering_plugin->htmlFormValuesAsArray('idno', null, false, false, true))) { $va_idno_values = array(); } $t_instance->set('idno', join($vs_sep, $va_idno_values)); // true=always set serial values, even if they already have a value; this let's us use the original pattern while replacing the serial value every time through break; } $t_instance->insert(); if ($t_instance->numErrors()) { $o_eventlog->log(array("CODE" => 'ERR', "SOURCE" => "mediaImport", "MESSAGE" => _t("Error creating new record while importing %1 from %2: %3", $f, $vs_relative_directory, join('; ', $t_instance->getErrors())))); $va_errors[$vs_relative_directory . '/' . $f] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $t_instance->getLabelForDisplay(), 'errors' => $t_instance->errors(), 'message' => $vs_msg = _t("Error creating new record while importing %1 from %2: %3", $f, $vs_relative_directory, join('; ', $t_instance->getErrors())), 'status' => 'ERROR'); $o_log->logError($vs_msg); $o_trans->rollback(); continue; } if ($t_instance->tableName() == 'ca_entities') { // entity labels deserve special treatment $t_instance->addLabel(array('surname' => $f), $vn_locale_id, null, true); } else { $t_instance->addLabel(array($t_instance->getLabelDisplayField() => $f), $vn_locale_id, null, true); } if ($t_instance->numErrors()) { $o_eventlog->log(array("CODE" => 'ERR', "SOURCE" => "mediaImport", "MESSAGE" => _t("Error creating record label while importing %1 from %2: %3", $f, $vs_relative_directory, join('; ', $t_instance->getErrors())))); $va_errors[$vs_relative_directory . '/' . $f] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $t_instance->getLabelForDisplay(), 'errors' => $t_instance->errors(), 'message' => $vs_msg = _t("Error creating record label while importing %1 from %2: %3", $f, $vs_relative_directory, join('; ', $t_instance->getErrors())), 'status' => 'ERROR'); $o_log->logError($vs_msg); $o_trans->rollback(); continue; } $t_new_rep = $t_instance->addRepresentation($vs_directory . '/' . $f, $vn_rep_type_id, $vn_locale_id, $vn_object_representation_status, $vn_object_representation_access, true, array('idno' => $vs_rep_idno), array('original_filename' => $f, 'returnRepresentation' => true, 'type_id' => $vn_rel_type_id)); if ($t_instance->numErrors()) { $o_eventlog->log(array("CODE" => 'ERR', "SOURCE" => "mediaImport", "MESSAGE" => _t("Error importing %1 from %2: ", $f, $vs_relative_directory, join('; ', $t_instance->getErrors())))); $va_errors[$vs_relative_directory . '/' . $f] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $t_instance->getLabelForDisplay(), 'errors' => $t_instance->errors(), 'message' => $vs_msg = _t("Error importing %1 from %2: %3", $f, $vs_relative_directory, join('; ', $t_instance->getErrors())), 'status' => 'ERROR'); $o_log->logError($vs_msg); $o_trans->rollback(); continue; } else { if ($vb_delete_media_on_import) { @unlink($vs_directory . '/' . $f); } } } } if ($t_instance->getPrimaryKey()) { // Perform import of embedded metadata (if required) if ($vn_mapping_id) { ca_data_importers::importDataFromSource($vs_directory . '/' . $f, $vn_mapping_id, array('logLevel' => $vs_log_level, 'format' => 'exif', 'forceImportForPrimaryKeys' => array($t_instance->getPrimaryKey(), 'transaction' => $o_trans))); } if ($vn_object_representation_mapping_id) { ca_data_importers::importDataFromSource($vs_directory . '/' . $f, $vn_object_representation_mapping_id, array('logLevel' => $vs_log_level, 'format' => 'exif', 'forceImportForPrimaryKeys' => array($t_new_rep->getPrimaryKey()), 'transaction' => $o_trans)); } $va_notices[$t_instance->getPrimaryKey()] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $t_instance->getLabelForDisplay(), 'message' => $vs_msg = _t('Imported %1 as %2', $f, $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD'))), 'status' => 'SUCCESS'); $o_log->logInfo($vs_msg); if ($vn_set_id) { $t_set->addItem($t_instance->getPrimaryKey(), null, $po_request->getUserID()); } $o_batch_log->addItem($t_instance->getPrimaryKey(), $t_instance->errors()); // Create relationships? if (is_array($va_create_relationship_for) && sizeof($va_create_relationship_for) && is_array($va_extracted_idnos_from_filename) && sizeof($va_extracted_idnos_from_filename)) { foreach ($va_extracted_idnos_from_filename as $vs_idno) { foreach ($va_create_relationship_for as $vs_rel_table) { if (!isset($va_relationship_type_id_for[$vs_rel_table]) || !$va_relationship_type_id_for[$vs_rel_table]) { continue; } $t_rel = $t_instance->getAppDatamodel()->getInstanceByTableName($vs_rel_table); if ($t_rel->load(array($t_rel->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno))) { $t_instance->addRelationship($vs_rel_table, $t_rel->getPrimaryKey(), $va_relationship_type_id_for[$vs_rel_table]); if (!$t_instance->numErrors()) { $va_notices[$t_instance->getPrimaryKey() . '_rel'] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $vs_label = $t_instance->getLabelForDisplay(), 'message' => $vs_msg = _t('Added relationship between <em>%1</em> and %2 <em>%3</em>', $vs_label, $t_rel->getProperty('NAME_SINGULAR'), $t_rel->getLabelForDisplay()), 'status' => 'RELATED'); $o_log->logInfo($vs_msg); } else { $va_notices[$t_instance->getPrimaryKey()] = array('idno' => $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), 'label' => $vs_label = $t_instance->getLabelForDisplay(), 'message' => $vs_msg = _t('Could not add relationship between <em>%1</em> and %2 <em>%3</em>: %4', $vs_label, $t_rel->getProperty('NAME_SINGULAR'), $t_rel->getLabelForDisplay(), join("; ", $t_instance->getErrors())), 'status' => 'ERROR'); $o_log->logError($vs_msg); } } } } } } else { $va_notices[$vs_relative_directory . '/' . $f] = array('idno' => '', 'label' => $f, 'message' => $vs_msg = $vs_import_mode == 'ALWAYS_MATCH' ? _t('Skipped %1 from %2 because it could not be matched', $f, $vs_relative_directory) : _t('Skipped %1 from %2', $f, $vs_relative_directory), 'status' => 'SKIPPED'); $o_log->logInfo($vs_msg); } if (isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) { $ps_callback($po_request, $vn_c, $vn_num_items, _t("[%3/%4] Processing %1 (%3)", caTruncateStringWithEllipsis($vs_relative_directory, 20) . '/' . caTruncateStringWithEllipsis($f, 30), $t_instance->get($t_instance->getProperty('ID_NUMBERING_ID_FIELD')), $vn_c, $vn_num_items), $t_new_rep, time() - $vn_start_time, memory_get_usage(true), $vn_c, sizeof($va_errors)); } $vn_c++; } if (isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) { $ps_callback($po_request, $vn_num_items, $vn_num_items, _t("Processing completed"), null, time() - $vn_start_time, memory_get_usage(true), $vn_c, sizeof($va_errors)); } $vn_elapsed_time = time() - $vn_start_time; if (isset($pa_options['reportCallback']) && ($ps_callback = $pa_options['reportCallback'])) { $va_general = array('elapsedTime' => $vn_elapsed_time, 'numErrors' => sizeof($va_errors), 'numProcessed' => $vn_c, 'batchSize' => $vn_num_items, 'table' => $t_instance->tableName(), 'set_id' => $t_set->getPrimaryKey(), 'setName' => $t_set->getLabelForDisplay()); $ps_callback($po_request, $va_general, $va_notices, $va_errors); } $o_batch_log->close(); if ($vb_we_set_transaction) { if (sizeof($va_errors) > 0) { $o_trans->rollback(); } else { $o_trans->commit(); } } $vs_set_name = $t_set->getLabelForDisplay(); $vs_started_on = caGetLocalizedDate($vn_start_time); if (isset($pa_options['sendMail']) && $pa_options['sendMail']) { if ($vs_email = trim($po_request->user->get('email'))) { caSendMessageUsingView($po_request, array($vs_email => $po_request->user->get('fname') . ' ' . $po_request->user->get('lname')), __CA_ADMIN_EMAIL__, _t('[%1] Batch media import completed', $po_request->config->get('app_display_name')), 'batch_media_import_completed.tpl', array('notices' => $va_notices, 'errors' => $va_errors, 'directory' => $vs_relative_directory, 'numErrors' => sizeof($va_errors), 'numProcessed' => $vn_c, 'subjectNameSingular' => _t('file'), 'subjectNamePlural' => _t('files'), 'startedOn' => $vs_started_on, 'completedOn' => caGetLocalizedDate(time()), 'setName' => $vn_set_id ? $vs_set_name : null, 'elapsedTime' => caFormatInterval($vn_elapsed_time))); } } if (isset($pa_options['sendSMS']) && $pa_options['sendSMS']) { SMS::send($po_request->getUserID(), _t("[%1] Media import processing for directory %2 with %3 %4 begun at %5 is complete", $po_request->config->get('app_display_name'), $vs_relative_directory, $vn_num_items, $vn_num_items == 1 ? _t('file') : _t('files'), $vs_started_on)); } $o_log->logInfo(_t("Media import processing for directory %1 with %2 %3 begun at %4 is complete", $vs_relative_directory, $vn_num_items, $vn_num_items == 1 ? _t('file') : _t('files'))); return array('errors' => $va_errors, 'notices' => $va_notices, 'processing_time' => caFormatInterval($vn_elapsed_time)); }