/**
*
*
* @param string $ps_refinery_name
*
* @return array
*/
function caGenericImportSplitter($ps_refinery_name, $ps_item_prefix, $ps_table, $po_refinery_instance, &$pa_destination_data, $pa_group, $pa_item, $pa_source_data, $pa_options)
{
global $g_ui_locale_id;
$po_refinery_instance->setReturnsMultipleValues(true);
$o_dm = Datamodel::load();
$po_refinery_instance->setReturnsMultipleValues(true);
$o_log = caGetOption('log', $pa_options, null);
$o_reader = caGetOption('reader', $pa_options, null);
$o_trans = caGetOption('transaction', $pa_options, null);
$pn_value_index = caGetOption('valueIndex', $pa_options, 0);
// We can probably always use the item destination – using group destination is a vestige of older code and no longer is used
// but we're leaving it in for now as a fallback it item dest is not set for some reason
$va_group_dest = isset($pa_item['destination']) && $pa_item['destination'] ? explode(".", $pa_item['destination']) : explode(".", $pa_group['destination']);
$vs_terminal = array_pop($va_group_dest);
$vs_dest_table = $va_group_dest[0];
$va_group_dest[] = $vs_terminal;
$pm_value = !isset($pa_source_data[$pa_item['source']]) && $o_reader ? caProcessImportItemSettingsForValue($o_reader->get($pa_item['source'], array('returnAsArray' => true)), $pa_item['settings']) : $pa_source_data[$pa_item['source']];
if (is_array($pm_value)) {
if (isset($pm_value[$pn_value_index])) {
$va_delimited_items = $pm_value[$pn_value_index];
// for input formats that support repeating values
} else {
$va_delimited_items = array_shift($va_delimited_items);
}
} else {
$va_delimited_items = array($pm_value);
}
if (!is_array($va_delimited_items)) {
$va_delimited_items = array($va_delimited_items);
}
$va_delimiter = $pa_item['settings']["{$ps_refinery_name}_delimiter"];
if (!is_array($va_delimiter)) {
$va_delimiter = array($va_delimiter);
}
if (sizeof($va_delimiter)) {
foreach ($va_delimiter as $vn_index => $vs_delim) {
if (!trim($vs_delim, "\t ")) {
unset($va_delimiter[$vn_index]);
continue;
}
$va_delimiter[$vn_index] = preg_quote($vs_delim, "!");
}
}
$va_match_on = caGetOption('matchOn', $pa_options, null);
if (!is_array($va_match_on) && $va_match_on) {
$va_match_on = array($va_match_on);
} elseif (is_array($va_match_on = $pa_item['settings']["{$ps_refinery_name}_matchOn"])) {
$pa_options['matchOn'] = $va_match_on;
}
$pb_dont_create = caGetOption('dontCreate', $pa_options, (bool) $pa_item['settings']["{$ps_refinery_name}_dontCreate"]);
$va_vals = array();
$vn_c = 0;
if (!($t_instance = $o_dm->getInstanceByTableName($ps_table, true))) {
return array();
}
if ($o_trans) {
$t_instance->setTransaction($o_trans);
}
$vs_label_fld = $t_instance->getLabelDisplayField();
if (sizeof($va_group_dest) == 1 && $vs_terminal == $ps_table || $vs_terminal != $ps_table && sizeof($va_group_dest) > 1) {
foreach ($va_delimited_items as $vn_x => $vs_delimited_item) {
$va_items = sizeof($va_delimiter) ? preg_split("!(" . join("|", $va_delimiter) . ")!", $vs_delimited_item) : array($vs_delimited_item);
foreach ($va_items as $vn_i => $vs_item) {
$va_parents = $pa_item['settings']["{$ps_refinery_name}_parents"];
if (!($vs_item = trim($vs_item))) {
if (is_array($va_parents) && sizeof($va_parents) > 0) {
// try to ladder up the parents hierarchy since the base value is blank (see PROV-972)
$vs_display_field = $t_instance->getLabelDisplayField();
while (sizeof($va_parents) > 0) {
$va_p = array_pop($va_parents);
if ($vs_laddered_val = BaseRefinery::parsePlaceholder($va_p[$vs_display_field], $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader, 'delimiter' => $va_delimiter, 'returnDelimitedValueAt' => $vn_x))) {
if ($o_log) {
$o_log->logDebug(_t('[{$ps_refinery_name}] Used parent value %1 because the mapped value was blank', $vs_item));
}
$vs_item = $vs_laddered_val;
break;
}
}
}
if (!$vs_item) {
continue;
}
}
if (is_array($va_skip_values = $pa_item['settings']["{$ps_refinery_name}_skipIfValue"]) && in_array($vs_item, $va_skip_values)) {
if ($o_log) {
$o_log->logDebug(_t('[{$ps_refinery_name}] Skipped %1 because it was in the skipIfValue list', $vs_item));
}
continue;
}
// Set label
$va_val = array();
// Set value as hierarchy
if ($va_hierarchy_setting = $pa_item['settings']["{$ps_refinery_name}_hierarchy"]) {
$va_attr_vals = $va_val = caProcessRefineryParents($ps_refinery_name, $ps_table, $va_hierarchy_setting, $pa_source_data, $pa_item, $pn_value_index, array_merge($pa_options, array('hierarchyMode' => true)));
$vs_item = $va_val['_preferred_label'];
} else {
// Set type
if ($vs_type_opt = $pa_item['settings']["{$ps_refinery_name}_{$ps_item_prefix}Type"]) {
$va_val['_type'] = BaseRefinery::parsePlaceholder($vs_type_opt, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader));
}
if ((!isset($va_val['_type']) || !$va_val['_type']) && ($vs_type_opt = $pa_item['settings']["{$ps_refinery_name}_{$ps_item_prefix}TypeDefault"])) {
if (!($va_val['_type'] = BaseRefinery::parsePlaceholder($vs_type_opt, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader, 'delimiter' => $va_delimiter, 'returnDelimitedValueAt' => $vn_x)))) {
$va_val['_type'] = BaseRefinery::parsePlaceholder($vs_type_opt, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader));
}
}
// Set lot_status
if ($vs_type_opt = $pa_item['settings']["{$ps_refinery_name}_{$ps_item_prefix}Status"]) {
$va_val['_status'] = BaseRefinery::parsePlaceholder($vs_type_opt, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader));
}
if ((!isset($va_val['_status']) || !$va_val['_status']) && ($vs_type_opt = $pa_item['settings']["{$ps_refinery_name}_{$ps_item_prefix}StatusDefault"])) {
$va_val['_status'] = BaseRefinery::parsePlaceholder($vs_type_opt, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader));
}
if ((!isset($va_val['_type']) || !$va_val['_type']) && $o_log) {
$o_log->logWarn(_t("[{$ps_refinery_name}] No %2 type is set for %2 %1", $vs_item, $ps_item_prefix));
}
//
// Storage location specific options
//
if ($ps_refinery_name == 'storageLocationSplitter' && ($va_hier_delimiter = $pa_item['settings']['storageLocationSplitter_hierarchicalDelimiter'])) {
if (!is_array($va_hier_delimiter)) {
$va_hier_delimiter = array($va_hier_delimiter);
}
if (sizeof($va_hier_delimiter)) {
foreach ($va_hier_delimiter as $vn_index => $vs_delim) {
if (!trim($vs_delim, "\t ")) {
unset($va_hier_delimiter[$vn_index]);
continue;
}
$va_hier_delimiter[$vn_index] = preg_quote($vs_delim, "!");
}
}
$va_location_hier = preg_split("!(" . join("|", $va_hier_delimiter) . ")!", $vs_item);
if (sizeof($va_location_hier) > 1) {
$vn_location_id = null;
if (!is_array($va_types = $pa_item['settings']['storageLocationSplitter_hierarchicalStorageLocationTypes'])) {
$va_types = array();
}
$vs_item = array_pop($va_location_hier);
if (!($va_val['_type'] = array_pop($va_types))) {
$va_val['_type'] = $pa_item['settings']['storageLocationSplitter_storageLocationTypeDefault'];
}
foreach ($va_location_hier as $vn_i => $vs_parent) {
if (sizeof($va_types) > 0) {
$vs_type = array_shift($va_types);
} else {
if (!($vs_type = $pa_item['settings']['storageLocationSplitter_storageLocationType'])) {
$vs_type = $pa_item['settings']['storageLocationSplitter_storageLocationTypeDefault'];
}
}
if (!$vs_type) {
break;
}
$vn_location_id = DataMigrationUtils::getStorageLocationID($vs_parent, $vn_location_id, $vs_type, $g_ui_locale_id, array('idno' => $vs_parent, 'parent_id' => $vn_location_id), $pa_options);
}
$va_val['parent_id'] = $va_val['_parent_id'] = $vn_location_id;
}
} else {
// Set parents
if ($va_parents) {
$va_val['parent_id'] = $va_val['_parent_id'] = caProcessRefineryParents($ps_refinery_name, $ps_table, $va_parents, $pa_source_data, $pa_item, $pn_value_index, $pa_options);
}
if (isset($pa_options['defaultParentID']) && (!isset($va_val['parent_id']) || !$va_val['parent_id'])) {
$va_val['parent_id'] = $va_val['_parent_id'] = $pa_options['defaultParentID'];
}
}
if (isset($pa_options['hierarchyID']) && $pa_options['hierarchyID'] && ($vs_hier_id_fld = $t_instance->getProperty('HIERARCHY_ID_FLD'))) {
$va_val[$vs_hier_id_fld] = $pa_options['hierarchyID'];
}
// Set attributes
if (is_array($va_attr_vals = caProcessRefineryAttributes($pa_item['settings']["{$ps_refinery_name}_attributes"], $pa_source_data, $pa_item, $vn_i, array('log' => $o_log, 'reader' => $o_reader)))) {
$va_val = array_merge($va_val, $va_attr_vals);
}
// Set interstitials
if (isset($pa_options['mapping']) && is_array($va_attr_vals = caProcessInterstitialAttributes($ps_refinery_name, $pa_options['mapping']->get('table_num'), $ps_table, $pa_source_data, $pa_item, $vn_i, array('log' => $o_log, 'reader' => $o_reader)))) {
$va_val = array_merge($va_val, $va_attr_vals);
}
// Set relationships on the related table
caProcessRefineryRelatedMultiple($po_refinery_instance, $pa_item, $pa_source_data, $vn_i, $o_log, $o_reader, $va_val, $va_attr_vals);
// Set nonpreferred labels
if (is_array($va_non_preferred_labels = $pa_item['settings']["{$ps_refinery_name}_nonPreferredLabels"])) {
$pa_options['nonPreferredLabels'] = array();
foreach ($va_non_preferred_labels as $va_label) {
foreach ($va_label as $vs_k => $vs_v) {
$va_label[$vs_k] = BaseRefinery::parsePlaceholder($vs_v, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader, 'returnAsString' => true, 'delimiter' => ' '));
}
$pa_options['nonPreferredLabels'][] = $va_label;
}
}
}
if ($vs_dest_table != $ps_table && sizeof($va_group_dest) > 1) {
$vs_item = BaseRefinery::parsePlaceholder($vs_item, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader, 'returnAsString' => true, 'delimiter' => ' '));
if (!is_array($va_attr_vals)) {
$va_attr_vals = array();
}
$va_attr_vals_with_parent = array_merge($va_attr_vals, array('parent_id' => $va_val['parent_id'] ? $va_val['parent_id'] : $va_val['_parent_id']));
$pa_options = array('matchOn' => array('idno', 'label')) + $pa_options;
switch ($ps_table) {
case 'ca_objects':
$vn_item_id = DataMigrationUtils::getObjectID($vs_item, $va_val['parent_id'], $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_object_lots':
if (isset($va_val['_status'])) {
$va_attr_vals['lot_status_id'] = $va_val['_status'];
}
unset($va_val['_status']);
$vn_item_id = DataMigrationUtils::getObjectLotID($vs_item, $vs_item, $va_val['_type'], $g_ui_locale_id, $va_attr_vals, $pa_options);
break;
case 'ca_entities':
$vn_item_id = DataMigrationUtils::getEntityID(DataMigrationUtils::splitEntityName($vs_item, $pa_options), $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_places':
$vn_item_id = DataMigrationUtils::getPlaceID($vs_item, $va_val['parent_id'], $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_occurrences':
$vn_item_id = DataMigrationUtils::getOccurrenceID($vs_item, $va_val['parent_id'], $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_collections':
$vn_item_id = DataMigrationUtils::getCollectionID($vs_item, $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_loans':
$vn_item_id = DataMigrationUtils::getLoanID($vs_item, $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_movements':
$vn_item_id = DataMigrationUtils::getMovementID($vs_item, $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_list_items':
if (!$pa_options['list_id']) {
if ($o_log) {
$o_log->logDebug(_t('[importHelpers:caGenericImportSplitter] List was not specified'));
}
continue 2;
}
$va_attr_vals_with_parent['is_enabled'] = 1;
$vn_item_id = DataMigrationUtils::getListItemID($pa_options['list_id'], $vs_item, $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_storage_locations':
$vn_item_id = DataMigrationUtils::getStorageLocationID($vs_item, $va_val['parent_id'], $va_val['_type'], $g_ui_locale_id, $va_attr_vals_with_parent, $pa_options);
break;
case 'ca_object_representations':
if ($o_log) {
$o_log->logDebug(_t('[importHelpers:caGenericImportSplitter] Only media paths can be mapped for object representations.'));
}
continue 2;
default:
if ($o_log) {
$o_log->logDebug(_t('[importHelpers:caGenericImportSplitter] Invalid table %1', $ps_table));
}
continue 2;
break;
}
if ($vn_item_id) {
$va_vals[][$vs_terminal] = $vn_item_id;
continue;
} else {
if ($o_log && !$pb_dont_create) {
$o_log->logError(_t("[{$ps_refinery_name}Refinery] Could not add %2 %1", $vs_item, $ps_item_prefix));
}
}
} elseif (sizeof($va_group_dest) == 1 && $vs_terminal == $ps_table) {
// Set relationship type
if ($vs_rel_type_opt = $pa_item['settings']["{$ps_refinery_name}_relationshipType"]) {
$va_val['_relationship_type'] = BaseRefinery::parsePlaceholder($vs_rel_type_opt, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader));
}
if ((!isset($va_val['_relationship_type']) || !$va_val['_relationship_type']) && ($vs_rel_type_opt = $pa_item['settings']["{$ps_refinery_name}_relationshipTypeDefault"])) {
if (!($va_val['_relationship_type'] = BaseRefinery::parsePlaceholder($vs_rel_type_opt, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader, 'delimiter' => $va_delimiter, 'returnDelimitedValueAt' => $vn_x)))) {
$va_val['_relationship_type'] = BaseRefinery::parsePlaceholder($vs_rel_type_opt, $pa_source_data, $pa_item, $pn_value_index, array('reader' => $o_reader));
}
}
if ((!isset($va_val['_relationship_type']) || !$va_val['_relationship_type']) && $o_log) {
$o_log->logWarn(_t("[{$ps_refinery_name}Refinery] No relationship type is set for %2 %1", $vs_item, $ps_item_prefix));
}
switch ($ps_table) {
case 'ca_entities':
$va_val['preferred_labels'] = DataMigrationUtils::splitEntityName($vs_item, $pa_options);
if (!isset($va_val['idno'])) {
$va_val['idno'] = $vs_item;
}
break;
case 'ca_list_items':
$va_val['preferred_labels'] = array('name_singular' => str_replace("_", " ", $vs_item), 'name_plural' => str_replace("_", " ", $vs_item));
$va_val['_list'] = $pa_options['list_id'];
if (!isset($va_val['idno'])) {
$va_val['idno'] = $vs_item;
}
break;
case 'ca_storage_locations':
case 'ca_movements':
case 'ca_loans':
case 'ca_collections':
case 'ca_occurrences':
case 'ca_places':
case 'ca_objects':
$va_val['preferred_labels'] = array('name' => $vs_item);
if (!isset($va_val['idno'])) {
$va_val['idno'] = $vs_item;
}
break;
case 'ca_object_lots':
$va_val['preferred_labels'] = array('name' => $vs_item);
if (!isset($va_val['idno_stub'])) {
$va_val['idno_stub'] = $vs_item;
}
if (isset($va_val['_status'])) {
$va_val['lot_status_id'] = $va_val['_status'];
}
unset($va_val['_status']);
break;
case 'ca_object_representations':
if (!($vs_batch_media_directory = $t_instance->getAppConfig()->get('batch_media_import_root_directory'))) {
break;
}
if (!isset($va_val['preferred_labels'])) {
$va_val['preferred_labels'] = array('name' => $vs_item);
}
if (isset($pa_item['settings']['objectRepresentationSplitter_mediaPrefix']) && $pa_item['settings']['objectRepresentationSplitter_mediaPrefix'] && isset($va_val['media']['media']) && $va_val['media']['media']) {
$va_val['media']['media'] = $vs_batch_media_directory . '/' . $pa_item['settings']['objectRepresentationSplitter_mediaPrefix'] . '/' . $va_val['media']['media'];
}
if (!isset($va_val['idno'])) {
$va_val['idno'] = $vs_item;
}
break;
default:
if ($o_log) {
$o_log->logDebug(_t('[importHelpers:caGenericImportSplitter] Invalid table %1', $ps_table));
}
continue 2;
break;
}
if (isset($pa_options['nonPreferredLabels']) && is_array($pa_options['nonPreferredLabels'])) {
$va_val['nonpreferred_labels'] = $pa_options['nonPreferredLabels'];
}
} elseif (sizeof($va_group_dest) == 2 && $vs_terminal == 'preferred_labels') {
switch ($ps_table) {
case 'ca_entities':
$va_val = DataMigrationUtils::splitEntityName($vs_item, $pa_options);
break;
case 'ca_list_items':
$va_val = array('name_singular' => $vs_item, 'name_plural' => $vs_item);
break;
case 'ca_storage_locations':
case 'ca_movements':
case 'ca_loans':
case 'ca_collections':
case 'ca_occurrences':
case 'ca_places':
case 'ca_objects':
$va_val = array('name' => $vs_item);
break;
case 'ca_object_lots':
$va_val = array('name' => $vs_item);
break;
case 'ca_object_representations':
if ($o_log) {
$o_log->logDebug(_t('[importHelpers:caGenericImportSplitter] Cannot map preferred labels to object representations. Only media paths can be mapped.'));
}
continue 2;
default:
if ($o_log) {
$o_log->logDebug(_t('[importHelpers:caGenericImportSplitter] Invalid table %1', $ps_table));
}
continue 2;
break;
}
} else {
if ($o_log) {
$o_log->logError(_t("[{$ps_refinery_name}Refinery] Could not add %2 %1: cannot map %3 using %1", $vs_item, $ps_item_prefix, join(".", $va_group_dest)));
}
}
$va_val['_matchOn'] = $va_match_on;
if ($pb_dont_create) {
$va_val['_dontCreate'] = 1;
}
$va_vals[] = $va_val;
$vn_c++;
}
}
} else {
if ($o_log) {
$o_log->logError(_t("[{$ps_refinery_name}Refinery] Cannot map %1 using this refinery", $pa_group['destination']));
}
return array();
}
return $va_vals;
}
/**
* Returns id for the row with the specified name (and type) or idno (regardless of specified type.) If the row does not already
* exist then it will be created with the specified name, type and locale, as well as with any specified values in the $pa_values array.
* $pa_values keys should be either valid entity fields or attributes.
*
* @param string $ps_table The table to match and/or create rows in
* @param array $pa_label Array with values for row label
* @param int $pn_parent_id
* @param int $pn_type_id The type_id or code of the type to use if the row needs to be created
* @param int $pn_locale_id The locale_id to use if the row needs to be created (will be used for both the row locale as well as the label locale)
* @param array $pa_values An optional array of additional values to populate newly created rows with. These values are *only* used for newly created rows; they will not be applied if the row named already exists unless the forceUpdate option is set, in which case attributes (but not intrinsics) will be updated. The array keys should be names of fields or valid attributes. Values should be either a scalar (for single-value attributes) or an array of values for (multi-valued attributes)
* @param array $pa_options An optional array of options, which include:
* outputErrors - if true, errors will be printed to console [default=false]
* dontCreate - if true then new entities will not be created [default=false]
* matchOn = optional list indicating sequence of checks for an existing record; values of array can be "label" and "idno". Ex. array("idno", "label") will first try to match on idno and then label if the first match fails. For entities only you may also specifiy "displayname", "surname" and "forename" to match on the text of the those label fields exclusively.
* matchOnDisplayName if true then entities are looked up exclusively using displayname, otherwise forename and surname fields are used [default=false]
* transaction - if Transaction instance is passed, use it for all Db-related tasks [default=null]
* returnInstance = return ca_entities instance rather than entity_id. Default is false.
* generateIdnoWithTemplate = A template to use when setting the idno. The template is a value with automatically-set SERIAL values replaced with % characters. Eg. 2012.% will set the created row's idno value to 2012.121 (assuming that 121 is the next number in the serial sequence.) The template is NOT used if idno is passed explicitly as a value in $pa_values.
* importEvent = if ca_data_import_events instance is passed then the insert/update of the entity will be logged as part of the import
* importEventSource = if importEvent is passed, then the value set for importEventSource is used in the import event log as the data source. If omitted a default value of "?" is used
* nonPreferredLabels = an optional array of nonpreferred labels to add to any newly created entities. Each label in the array is an array with required entity label values.
* forceUpdate = update attributes set in $pa_values even if row already exists. [Default=false; no values are updated in existing rows]
* matchMediaFilesWithoutExtension = For ca_object_representations, if media path is invalid, attempt to find media in referenced directory and sub-directories that has a matching name, regardless of file extension. [default=false]
* log = if KLogger instance is passed then actions will be logged
* ignoreParent = Don't take into account parent_id value when looking for matching rows [Default is false]
* @return bool|BaseModel|mixed|null
*/
private static function _getID($ps_table, $pa_label, $pn_parent_id, $pn_type_id, $pn_locale_id, $pa_values = null, $pa_options = null)
{
if (!is_array($pa_options)) {
$pa_options = array();
}
$o_dm = Datamodel::load();
/** @var KLogger $o_log */
$o_log = isset($pa_options['log']) && $pa_options['log'] instanceof KLogger ? $pa_options['log'] : null;
if (!($t_instance = $o_dm->getInstanceByTableName($ps_table, true))) {
return null;
}
$vs_table_display_name = $t_instance->getProperty('NAME_SINGULAR');
$vs_table_class = $t_instance->tableName();
$vs_label_display_fld = $t_instance->getLabelDisplayField();
$vs_label = $pa_label[$vs_label_display_fld];
$pb_output_errors = caGetOption('outputErrors', $pa_options, false);
$pb_match_on_displayname = caGetOption('matchOnDisplayName', $pa_options, false);
$pa_match_on = caGetOption('matchOn', $pa_options, array('label', 'idno', 'displayname'), array('castTo' => "array"));
$ps_event_source = caGetOption('importEventSource', $pa_options, '?');
$pb_match_media_without_ext = caGetOption('matchMediaFilesWithoutExtension', $pa_options, false);
$pb_ignore_parent = caGetOption('ignoreParent', $pa_options, false);
$vn_parent_id = $pn_parent_id ? $pn_parent_id : caGetOption('parent_id', $pa_values, null);
if (!$vn_parent_id) {
$vn_parent_id = null;
}
$vs_idno_fld = $t_instance->getProperty('ID_NUMBERING_ID_FIELD');
$vs_idno = caGetOption($vs_idno_fld, $pa_values, null);
/** @var ca_data_import_events $o_event */
$o_event = isset($pa_options['importEvent']) && $pa_options['importEvent'] instanceof ca_data_import_events ? $pa_options['importEvent'] : null;
if (isset($pa_options['transaction']) && $pa_options['transaction'] instanceof Transaction) {
$t_instance->setTransaction($pa_options['transaction']);
if ($o_event) {
$o_event->setTransaction($pa_options['transaction']);
}
}
if (preg_match('!\\%!', $vs_idno)) {
$pa_options['generateIdnoWithTemplate'] = $vs_idno;
$vs_idno = null;
}
if (!$vs_idno) {
if (isset($pa_options['generateIdnoWithTemplate']) && $pa_options['generateIdnoWithTemplate']) {
$pa_values[$vs_idno_fld] = $vs_idno = $t_instance->setIdnoWithTemplate($pa_options['generateIdnoWithTemplate'], array('dontSetValue' => true));
}
}
$va_regex_list = $va_replacements_list = null;
if ($vs_table_class == 'ca_object_representations') {
// Get list of regular expressions that user can use to transform file names to match object idnos
$va_regex_list = caBatchGetMediaFilenameToIdnoRegexList(array('log' => $o_log));
// Get list of replacements that user can use to transform file names to match object idnos
$va_replacements_list = caBatchGetMediaFilenameReplacementRegexList(array('log' => $o_log));
}
$vn_id = null;
foreach ($pa_match_on as $vs_match_on) {
switch (strtolower($vs_match_on)) {
case 'idno':
if ($vs_idno == '%') {
break;
}
// don't try to match on an unreplaced idno placeholder
switch ($vs_table_class) {
case 'ca_object_representations':
//
// idno lookups for representations use media batch importer rules
//
$va_idnos_to_match = array($vs_idno);
if (is_array($va_replacements_list)) {
foreach ($va_replacements_list as $vs_replacement_code => $va_replacement) {
if (isset($va_replacement['search']) && is_array($va_replacement['search'])) {
$va_replace = caGetOption('replace', $va_replacement);
$va_search = array();
foreach ($va_replacement['search'] as $vs_search) {
$va_search[] = '!' . $vs_search . '!';
}
if ($vs_idno_proc = @preg_replace($va_search, $va_replace, $vs_idno)) {
$va_idnos_to_match[] = $vs_idno_proc;
}
}
}
}
if (is_array($va_regex_list) && sizeof($va_regex_list)) {
foreach ($va_regex_list as $vs_regex_name => $va_regex_info) {
foreach ($va_regex_info['regexes'] as $vs_regex) {
foreach ($va_idnos_to_match as $vs_idno_match) {
if (!$vs_idno_match) {
continue;
}
if (preg_match('!' . $vs_regex . '!', $vs_idno_match, $va_matches)) {
if ($vn_id = ca_object_representations::find(array('idno' => $va_matches[1]), array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction']))) {
break 5;
}
}
}
}
}
} else {
foreach ($va_idnos_to_match as $vs_idno_match) {
if (!$vs_idno_match) {
continue;
}
if ($vn_id = ca_object_representations::find(array('idno' => $vs_idno_match), array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction']))) {
break 3;
}
}
}
break;
default:
//
// Standard idno lookup for most tables
//
$va_find_vals = array($vs_idno_fld => $vs_idno ? $vs_idno : ($pa_label['_originalText'] ? $pa_label['_originalText'] : $vs_label), 'type_id' => $pn_type_id);
if (!$pb_ignore_parent && isset($pa_values['parent_id'])) {
$va_find_vals['parent_id'] = $pa_values['parent_id'];
}
if (($vs_idno || trim($pa_label['_originalText'] || $vs_label)) && ($vn_id = $vs_table_class::find($va_find_vals, array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction'])))) {
break 2;
}
break;
}
break;
case 'label':
case 'labels':
if ($pb_match_on_displayname && strlen(trim($pa_label['displayname'])) > 0) {
// entities only
$va_params = array('preferred_labels' => array('displayname' => $pa_label['displayname']), 'type_id' => $pn_type_id);
if (!$pb_ignore_parent && isset($pa_values['parent_id'])) {
$va_params['parent_id'] = $vn_parent_id;
}
$vn_id = $vs_table_class::find($va_params, array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction']));
} elseif ($vs_table_class == 'ca_entities') {
// entities only
$va_params = array('preferred_labels' => array('forename' => $pa_label['forename'], 'middlename' => $pa_label['middlename'], 'surname' => $pa_label['surname']), 'type_id' => $pn_type_id);
if (!$pb_ignore_parent) {
$va_params['parent_id'] = $vn_parent_id;
}
$vn_id = $vs_table_class::find($va_params, array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction']));
} else {
$va_params = array('preferred_labels' => array($vs_label_display_fld => $pa_label[$vs_label_display_fld]), 'type_id' => $pn_type_id);
if (!$pb_ignore_parent && isset($pa_values['parent_id'])) {
$va_params['parent_id'] = $vn_parent_id;
}
$vn_id = $vs_table_class::find($va_params, array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction']));
}
if ($vn_id) {
break 2;
}
break;
//
// For entities only
//
//
// For entities only
//
case 'surname':
$va_params = array('preferred_labels' => array('surname' => $pa_label['surname']), 'type_id' => $pn_type_id);
if (!$pb_ignore_parent && isset($pa_values['parent_id'])) {
$va_params['parent_id'] = $vn_parent_id;
}
$vn_id = $vs_table_class::find($va_params, array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction']));
if ($vn_id) {
break 2;
}
break;
case 'forename':
$va_params = array('preferred_labels' => array('forename' => $pa_label['forename']), 'type_id' => $pn_type_id);
if (!$pb_ignore_parent && isset($pa_values['parent_id'])) {
$va_params['parent_id'] = $vn_parent_id;
}
$vn_id = $vs_table_class::find($va_params, array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction']));
if ($vn_id) {
break 2;
}
break;
case 'displayname':
$va_params = array('preferred_labels' => array('displayname' => $pa_label['displayname']), 'type_id' => $pn_type_id);
if (!$pb_ignore_parent && isset($pa_values['parent_id'])) {
$va_params['parent_id'] = $vn_parent_id;
}
$vn_id = $vs_table_class::find($va_params, array('returnAs' => 'firstId', 'purifyWithFallback' => true, 'transaction' => $pa_options['transaction']));
if ($vn_id) {
break 2;
}
break;
}
}
if (!$vn_id) {
//
// Create new row
//
if (caGetOption('dontCreate', $pa_options, false)) {
return false;
}
if ($o_event) {
$o_event->beginItem($ps_event_source, $vs_table_class, 'I');
}
// If we're creating a new item, it's probably a good idea to *NOT* use a
// BaseModel instance from cache, because those cannot change their type_id
if (!($t_instance = $o_dm->getInstanceByTableName($ps_table, false))) {
return null;
}
if (isset($pa_options['transaction']) && $pa_options['transaction'] instanceof Transaction) {
$t_instance->setTransaction($pa_options['transaction']);
}
$t_instance->setMode(ACCESS_WRITE);
$t_instance->set('locale_id', $pn_locale_id);
$t_instance->set('type_id', $pn_type_id);
$va_intrinsics = array('source_id' => null, 'access' => 0, 'status' => 0, 'lifespan' => null, 'parent_id' => $vn_parent_id, 'lot_status_id' => null, '_interstitial' => null);
if ($vs_hier_id_fld = $t_instance->getProperty('HIERARCHY_ID_FLD')) {
$va_intrinsics[$vs_hier_id_fld] = null;
}
if ($vs_idno_fld) {
$va_intrinsics[$vs_idno_fld] = $vs_idno ? $vs_idno : null;
}
foreach ($va_intrinsics as $vs_fld => $vm_fld_default) {
if ($t_instance->hasField($vs_fld)) {
$t_instance->set($vs_fld, caGetOption($vs_fld, $pa_values, $vm_fld_default));
}
unset($pa_values[$vs_fld]);
}
if ($t_instance->hasField('media') && $t_instance->getFieldInfo('media', 'FIELD_TYPE') == FT_MEDIA && isset($pa_values['media']) && $pa_values['media']) {
if (is_array($pa_values['media'])) {
$pa_values['media'] = array_shift($pa_values['media']);
}
if ($pb_match_media_without_ext && !isURL($pa_values['media']) && !file_exists($pa_values['media'])) {
$vs_dirname = pathinfo($pa_values['media'], PATHINFO_DIRNAME);
$vs_filename = preg_replace('!\\.[A-Za-z0-9]{1,4}$!', '', pathinfo($pa_values['media'], PATHINFO_BASENAME));
$vs_original_path = $pa_values['media'];
$pa_values['media'] = null;
$va_files_in_dir = caGetDirectoryContentsAsList($vs_dirname, true, false, false, false);
foreach ($va_files_in_dir as $vs_filepath) {
if ($o_log) {
$o_log->logDebug(_t("Trying media %1 in place of %2/%3", $vs_filepath, $vs_original_path, $vs_filename));
}
if (pathinfo($vs_filepath, PATHINFO_FILENAME) == $vs_filename) {
if ($o_log) {
$o_log->logNotice(_t("Found media %1 for %2/%3", $vs_filepath, $vs_original_path, $vs_filename));
}
$pa_values['media'] = $vs_filepath;
break;
}
}
}
$t_instance->set('media', $pa_values['media']);
}
$t_instance->insert();
if ($t_instance->numErrors()) {
if ($pb_output_errors) {
print "[Error] " . _t("Could not insert %1 %2: %3", $vs_table_display_name, $pa_label[$vs_label_display_fld], join('; ', $t_instance->getErrors())) . "\n";
}
if ($o_log) {
$o_log->logError(_t("Could not insert %1 %2: %3", $vs_table_display_name, $pa_label[$vs_label_display_fld], join('; ', $t_instance->getErrors())));
}
return null;
}
$vb_label_errors = false;
$t_instance->addLabel($pa_label, $pn_locale_id, null, true);
if ($t_instance->numErrors()) {
if ($pb_output_errors) {
print "[Error] " . _t("Could not set preferred label for %1 %2: %3", $vs_table_display_name, $pa_label[$vs_label_display_fld], join('; ', $t_instance->getErrors())) . "\n";
}
if ($o_log) {
$o_log->logError(_t("Could not set preferred label for %1 %2: %3", $vs_table_display_name, $pa_label[$vs_label_display_fld], join('; ', $t_instance->getErrors())));
}
$vb_label_errors = true;
}
DataMigrationUtils::_setIdno($t_instance, $vs_idno, $pa_options);
$vb_attr_errors = !DataMigrationUtils::_setAttributes($t_instance, $pn_locale_id, $pa_values, $pa_options);
DataMigrationUtils::_setNonPreferredLabels($t_instance, $pn_locale_id, $pa_options);
$vn_id = $t_instance->getPrimaryKey();
if ($o_event) {
if ($vb_attr_errors || $vb_label_errors) {
$o_event->endItem($vn_id, __CA_DATA_IMPORT_ITEM_PARTIAL_SUCCESS__, _t("Errors setting field values: %1", join('; ', $t_instance->getErrors())));
} else {
$o_event->endItem($vn_id, __CA_DATA_IMPORT_ITEM_SUCCESS__, '');
}
}
if ($o_log) {
$o_log->logInfo(_t("Created new %1 %2", $vs_table_display_name, $pa_label[$vs_label_display_fld]));
}
if (isset($pa_options['returnInstance']) && $pa_options['returnInstance']) {
return $t_instance;
}
} else {
if ($o_event) {
$o_event->beginItem($ps_event_source, $vs_table_class, 'U');
}
if ($o_log) {
$o_log->logDebug(_t("Found existing %1 %2 in DataMigrationUtils::_getID()", $vs_table_display_name, $pa_label[$vs_label_display_fld]));
}
$vb_attr_errors = false;
if (($vb_force_update = caGetOption('forceUpdate', $pa_options, false)) || ($vb_return_instance = caGetOption('returnInstance', $pa_options, false))) {
if (!($t_instance = $o_dm->getInstanceByTableName($vs_table_class, false))) {
return null;
}
if (isset($pa_options['transaction']) && $pa_options['transaction'] instanceof Transaction) {
$t_instance->setTransaction($pa_options['transaction']);
}
$vb_has_attr = false;
if ($vb_force_update) {
foreach ($pa_values as $vs_element => $va_values) {
if ($t_instance->hasElement($vs_element)) {
$vb_has_attr = true;
break;
}
}
}
if ($vb_return_instance || $vb_force_update && $vb_has_attr) {
$vn_rc = $t_instance->load($vn_id);
} else {
$vn_rc = true;
}
if (!$vn_rc) {
if ($o_log) {
$o_log->logError(_t("Could not load existing %1 with id %2 (%3) in DataMigrationUtils::_getID() [THIS SHOULD NOT HAPPEN]", $vs_table_display_name, $vn_id, $pa_label[$vs_label_display_fld]));
}
return null;
} else {
if ($vb_force_update && $vb_has_attr) {
if ($vb_attr_errors = !DataMigrationUtils::_setAttributes($t_instance, $pn_locale_id, $pa_values, $pa_options)) {
if ($o_log) {
$o_log->logError(_t("Could not set attributes for %1 with id %2 (%3) in DataMigrationUtils::_getID(): %4", $vs_table_display_name, $vn_id, $pa_label[$vs_label_display_fld], join("; ", $t_instance->getErrors())));
}
}
}
if ($o_event) {
if ($vb_attr_errors) {
$o_event->endItem($vn_id, __CA_DATA_IMPORT_ITEM_PARTIAL_SUCCESS__, _t("Errors setting field values: %1", join('; ', $t_instance->getErrors())));
} else {
$o_event->endItem($vn_id, __CA_DATA_IMPORT_ITEM_SUCCESS__, '');
}
}
if ($vb_return_instance) {
return $t_instance;
}
}
}
if ($o_event) {
$o_event->endItem($vn_id, __CA_DATA_IMPORT_ITEM_SUCCESS__, '');
}
}
return $vn_id;
}
/**
* Import oral histories from specified directory into CollectiveAccess database
*/
public function commandImportOralHistories()
{
$o_conf = $this->getToolConfig();
// Get locale from config and translate to numeric code
$t_locale = new ca_locales();
$pn_locale_id = $t_locale->localeCodeToID($o_conf->get('locale'));
$o_log = $this->getLogger();
$o_progress = $this->getProgressBar(0);
$vs_transcript_dir = $this->getSetting("transcript_directory");
if (!is_readable($vs_transcript_dir)) {
if ($o_log) {
$o_log->logError($vs_err_msg = _t("Transcript directory %1 is not readable", $vs_transcript_dir));
}
if ($o_progress) {
$o_progress->setError($vs_err_msg);
$o_progress->finish();
}
return false;
}
$vs_audio_dir = $this->getSetting("audio_directory");
if (!is_readable($vs_audio_dir)) {
if ($o_log) {
$o_log->logError($vs_err_msg = _t("Audio directory %1 is not readable", $vs_audio_dir));
}
if ($o_progress) {
$o_progress->setError($vs_err_msg);
$o_progress->finish();
}
return false;
}
if ($o_progress) {
$o_progress->start("Starting oral history import");
}
// ----------------------------------------------------------------------
// process main data
$r_dir = opendir($vs_transcript_dir);
while (($vs_file = readdir($r_dir)) !== false) {
if ($vs_file[0] == '.') {
continue;
}
// Get markup and fix it up to be valid XML
$vs_markup = file_get_contents($vs_transcript_dir . $vs_file);
$vs_markup = preg_replace('!&!', '&', $vs_markup);
$vs_xml = "<?xml version=\"1.0\" encoding=\"utf-8\"?><transcript>{$vs_markup}</transcript>";
try {
$o_xml = new SimpleXMLElement($vs_xml);
} catch (Exception $e) {
$o_log->logError("Could not parse XML transcript for {$vs_transcript_dir}{$vs_file}: " . $e->getMessage());
continue;
}
$vs_idno = (string) $o_xml->identifier;
if (!file_exists($vs_media_path = "{$vs_audio_dir}{$vs_idno}.mp3")) {
$o_log->logError("No audio file found for {$vs_idno}. File path was {$vs_media_path}");
continue;
}
$vs_title = (string) $o_xml->title;
$vs_date_created = (string) $o_xml->datecreated;
$vs_format = (string) $o_xml->format;
$vs_medium = (string) $o_xml->medium;
$vs_place_recorded = (string) $o_xml->placeRecorded;
$vs_rights = (string) $o_xml->rights;
$vs_extent = (string) $o_xml->extent;
$vs_country = (string) $o_xml->countryOfOrigin;
$va_interviewers = array();
foreach ($o_xml->interviewer as $o_interviewer) {
$va_interviewers[(string) $o_interviewer->attributes()->abbreviation] = (string) $o_interviewer;
}
$va_participants = array();
foreach ($o_xml->participant as $o_participant) {
$va_participants[(string) $o_participant->attributes()->abbreviation] = (string) $o_participant;
}
$va_observers = array();
if ($o_xml->observer) {
foreach ($o_xml->observer as $o_observer) {
$va_observers[] = (string) $o_observer;
}
}
// Create object
$t_object = new ca_objects();
$t_object->setMode(ACCESS_WRITE);
if (!$t_object->load(array('idno' => $vs_idno, 'deleted' => 0))) {
$t_object->set('type_id', 'oral_history');
$t_object->set('idno', $vs_idno);
$t_object->set('status', 0);
$t_object->set('access', 1);
}
$t_object->addAttribute(array('locale_id' => $pn_locale_id, 'dc_format' => $vs_format), 'dc_format');
$t_object->addAttribute(array('locale_id' => $pn_locale_id, 'dates_value' => $vs_date_created, 'dc_dates_types' => 'created'), 'date');
$t_object->addAttribute(array('locale_id' => $pn_locale_id, 'medium' => $vs_medium), 'medium');
$t_object->addAttribute(array('locale_id' => $pn_locale_id, 'interview_location' => $vs_place_recorded), 'interview_location');
$t_object->addAttribute(array('locale_id' => $pn_locale_id, 'rights' => $vs_rights), 'rights');
$t_object->addAttribute(array('locale_id' => $pn_locale_id, 'extent' => $vs_extent), 'extent');
$t_object->addAttribute(array('locale_id' => $pn_locale_id, 'countryOfOrigin' => $vs_country), 'countryOfOrigin');
if (!$t_object->getPrimaryKey()) {
$t_object->insert();
DataMigrationUtils::postError($t_object, 'While inserting object');
if ($t_object->numErrors()) {
$o_log->logError("While adding object for {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_object->getErrors()));
}
$t_object->addLabel(array('name' => $vs_title), $pn_locale_id, null, true);
DataMigrationUtils::postError($t_object, 'While adding object label');
if ($t_object->numErrors()) {
$o_log->logError("While adding object label for {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_object->getErrors()));
}
} else {
$t_object->update();
DataMigrationUtils::postError($t_object, 'While updating object');
if ($t_object->numErrors()) {
$o_log->logError("While updating object for {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_object->getErrors()));
}
}
// add entities
foreach ($va_interviewers as $vs_abbr => $vs_name) {
$vn_entity_id = DataMigrationUtils::getEntityID(DataMigrationUtils::splitEntityName($vs_name), 'ind', $pn_locale_id);
$t_object->addRelationship('ca_entities', $vn_entity_id, 'interviewer');
DataMigrationUtils::postError($t_object, "While adding interviewer {$vs_name} to object");
if ($t_object->numErrors()) {
$o_log->logError("While adding interview {$vs_name} to {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_object->getErrors()));
}
}
foreach ($va_participants as $vs_abbr => $vs_name) {
$vn_entity_id = DataMigrationUtils::getEntityID(DataMigrationUtils::splitEntityName($vs_name), 'ind', $pn_locale_id);
$t_object->addRelationship('ca_entities', $vn_entity_id, 'interviewee');
DataMigrationUtils::postError($t_object, "While adding interviewee {$vs_name} to object");
if ($t_object->numErrors()) {
$o_log->logError("While adding interviee {$vs_name} to {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_object->getErrors()));
}
}
foreach ($va_observers as $vn_i => $vs_name) {
$vn_entity_id = DataMigrationUtils::getEntityID(DataMigrationUtils::splitEntityName($vs_name), 'ind', $pn_locale_id);
$t_object->addRelationship('ca_entities', $vn_entity_id, 'observer');
DataMigrationUtils::postError($t_object, "While adding observer {$vs_name} to object");
if ($t_object->numErrors()) {
$o_log->logError("While adding observer {$vs_name} to {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_object->getErrors()));
}
}
// Add media
$t_rep = $t_object->addRepresentation($vs_media_path, "front", $pn_locale_id, 0, 1, true, array(), array('returnRepresentation' => true));
DataMigrationUtils::postError($t_object, "While adding representation {$vs_media_path} to object");
if ($t_object->numErrors()) {
$o_log->logError("While adding representation {$vs_media_path} to {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_object->getErrors()));
}
if ($t_object->numErrors()) {
continue;
}
$va_clips = array();
foreach ($o_xml->clip as $o_clip) {
$vs_content = nl2br(preg_replace('!^[\\n\\r\\t]+!', '', trim((string) $o_clip->asXML())));
$vs_start = (string) $o_clip->attributes()->start;
$va_themes = $va_places = array();
foreach ($o_clip->children() as $o_node) {
$vs_tag = (string) $o_node->getName();
switch ($vs_tag) {
case 'place':
$va_places[] = (string) $o_node;
break;
default:
$va_themes[] = $vs_tag;
break;
}
}
$va_clips[] = array('start' => $vs_start, 'content' => $vs_content, 'themes' => $va_themes, 'places' => $va_places);
}
foreach ($va_clips as $vn_i => $va_clip) {
$vs_start = $va_clip['start'];
if (!($vs_end = $va_clips[$vn_i + 1]['start'])) {
$va_info = $t_rep->getMediaInfo('media', 'original');
$vs_end = $va_info['PROPERTIES']['duration'];
}
//print "[$vs_start/$vs_end] (".join('/', $va_clip['themes'])."); (".join('/', $va_clip['places']).") ".substr($va_clip['content'], 0, 30)."\n\n\n";
$t_annotation = $t_rep->addAnnotation("{$vs_start} ... {$vs_end}", $pn_locale_id, 1, array('startTimecode' => $vs_start, 'endTimecode' => $vs_end), 0, 1, array('transcription' => $va_clip['content']), array('returnAnnotation' => true));
DataMigrationUtils::postError($t_rep, "While adding annotation to representation");
if ($t_rep->numErrors()) {
$o_log->logError("While adding annotation {$vs_start}/{$vs_end} to {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_rep->getErrors()));
}
if ($t_annotation) {
foreach ($va_clip['themes'] as $vs_theme) {
$t_annotation->addRelationship('ca_list_items', $vs_theme, 'describes');
DataMigrationUtils::postError($t_annotation, "While adding theme {$vs_theme} to annotation");
if ($t_annotation->numErrors()) {
$o_log->logError("While adding theme {$vs_theme} to annotation {$vs_start}/{$vs_end} for {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_annotation->getErrors()));
}
}
foreach ($va_clip['places'] as $vs_place) {
if ($vn_place_id = ca_places::find(array('preferred_labels' => array('name' => $vs_place)), array('returnAs' => 'firstId'))) {
$t_annotation->addRelationship('ca_places', $vn_place_id, 'describes');
DataMigrationUtils::postError($t_annotation, "While adding place {$vs_place} to annotation");
if ($t_annotation->numErrors()) {
$o_log->logError("While adding place {$vs_place} to annotation {$vs_start}/{$vs_end} for {$vs_transcript_dir}{$vs_file}: " . join("; ", $t_annotation->getErrors()));
}
}
}
}
}
$o_log->logInfo("Imported {$vs_file}");
}
$o_progress->finish("Completed processing");
if ($o_log) {
$o_log->logDebug(_t("Ended oral history import"));
}
return true;
}
switch ($vs_np_term_type) {
default:
$vn_np_term_type_id = null;
break;
}
if (!$t_place->addLabel(array('name' => $vs_np_label, 'description' => ''), $pn_en_locale_id, $vn_np_term_type_id, false)) {
print "[Error] Could not add non-preferred label to TGN term [" . $va_subject['subject_id'] . "] " . $vs_np_label . ": " . join("; ", $t_place->getErrors()) . "\n";
}
}
}
// Add place types
if ($vs_place_type_relationship_code && $va_subject['place_type_id']) {
$va_tmp = explode('/', $va_subject['place_type_id']);
if ($vn_item_id = DataMigrationUtils::getListItemID('tgn_place_types', $va_tmp[0], null, $pn_en_locale_id, array('name_singular' => $va_tmp[1], 'name_plural' => $va_tmp[1]), array())) {
$t_place->addRelationship('ca_list_items', $vn_item_id, $vs_place_type_relationship_code, null, null, null, null, array('allowDuplicates' => true));
DataMigrationUtils::postError($t_place, "[Error] While adding place type to place");
}
}
$vn_term_count++;
} else {
print "[Error] Could not import TGN term [" . $va_subject['subject_id'] . "] " . $vs_preferred_term . ": " . join("; ", $t_list->getErrors()) . "\n";
}
} else {
$va_subject = array('subject_id' => $o_xml->getAttribute('Subject_ID'));
$va_coords = array();
}
break;
# ---------------------------
# ---------------------------
case 'Descriptive_Note':
if ($o_xml->nodeType == XMLReader::ELEMENT) {
print $vs_form_name;
?>
Errors<?php
print $vs_field_name_prefix . $vs_n;
?>
"> </div>
<?php
$va_force_new_label = array();
foreach ($t_subject->getLabelUIFields() as $vn_i => $vs_fld) {
$va_force_new_label[$vs_fld] = '';
}
$va_force_new_label['locale_id'] = $g_ui_locale_id;
// use default locale
$va_force_new_label[$t_subject->getLabelDisplayField()] = $vs_q;
// query text is used for display field
$va_force_new_label = array_merge($va_force_new_label, DataMigrationUtils::splitEntityName($vs_q));
// query text split as entity name used for other entity label fields
$va_form_elements = $t_subject->getBundleFormHTMLForScreen($this->getVar('screen'), array('request' => $this->request, 'formName' => $vs_form_name . $vs_field_name_prefix . $vs_n, 'forceLabelForNew' => $va_force_new_label));
print join("\n", $va_form_elements);
?>
<input type='hidden' name='_formName' value='<?php
print $vs_form_name . $vs_field_name_prefix . $vs_n;
?>
'/>
<input type='hidden' name='q' value='<?php
print htmlspecialchars($vs_q, ENT_QUOTES, 'UTF-8');
?>
'/>
<input type='hidden' name='screen' value='<?php
print htmlspecialchars($this->getVar('screen'));
?>
/**
* Sets the source text encoding to be used by DataMigrationUtils::transformTextEncoding()
*/
static function setSourceTextEncoding($ps_encoding)
{
DataMigrationUtils::$s_source_encoding = $ps_encoding;
}
/**
*
*
* @param string $ps_source
* @param string $ps_mapping
* @param array $pa_options
* user_id = user to execute import for
* description = Text describing purpose of import to be logged.
* showCLIProgressBar = Show command-line progress bar. Default is false.
* format = Format of data being imported. MANDATORY
* useNcurses = Use ncurses library to format output
* logDirectory = path to directory where logs should be written
* logLevel = KLogger constant for minimum log level to record. Default is KLogger::INFO. Constants are, in descending order of shrillness:
* KLogger::EMERG = Emergency messages (system is unusable)
* KLogger::ALERT = Alert messages (action must be taken immediately)
* KLogger::CRIT = Critical conditions
* KLogger::ERR = Error conditions
* KLogger::WARN = Warnings
* KLogger::NOTICE = Notices (normal but significant conditions)
* KLogger::INFO = Informational messages
* KLogger::DEBUG = Debugging messages
* dryRun = do import but don't actually save data
* environment = an array of environment values to provide to the import process. The keys manifest themselves as mappable tags.
* forceImportForPrimaryKeys = list of primary key ids to force mapped source data into. The number of keys passed should equal or exceed the number of rows in the source data. [Default is empty]
* transaction = transaction to perform import within. Will not be used if noTransaction option is set. [Default is to create a new transaction]
* noTransaction = don't wrap the import in a transaction. [Default is false]
*/
public static function importDataFromSource($ps_source, $ps_mapping, $pa_options = null)
{
ca_data_importers::$s_num_import_errors = 0;
ca_data_importers::$s_num_records_processed = 0;
ca_data_importers::$s_num_records_skipped = 0;
ca_data_importers::$s_import_error_list = array();
$opa_app_plugin_manager = new ApplicationPluginManager();
$va_notices = $va_errors = array();
$pb_no_transaction = caGetOption('noTransaction', $pa_options, false, array('castTo' => 'bool'));
$pa_force_import_for_primary_keys = caGetOption('forceImportForPrimaryKeys', $pa_options, null);
if (!($t_mapping = ca_data_importers::mappingExists($ps_mapping))) {
return null;
}
$o_event = ca_data_import_events::newEvent(isset($pa_options['user_id']) ? $pa_options['user_id'] : null, $pa_options['format'], $ps_source, isset($pa_options['description']) ? $pa_options['description'] : '');
$o_trans = null;
if (!$pb_no_transaction) {
if (!($o_trans = caGetOption('transaction', $pa_options, null))) {
$o_trans = new Transaction();
}
$t_mapping->setTransaction($o_trans);
}
$po_request = caGetOption('request', $pa_options, null);
$pb_dry_run = caGetOption('dryRun', $pa_options, false);
$pn_file_number = caGetOption('fileNumber', $pa_options, 0);
$pn_number_of_files = caGetOption('numberOfFiles', $pa_options, 1);
$o_config = Configuration::load();
if (!is_array($pa_options) || !isset($pa_options['logLevel']) || !$pa_options['logLevel']) {
$pa_options['logLevel'] = KLogger::INFO;
}
if (!is_array($pa_options) || !isset($pa_options['logDirectory']) || !$pa_options['logDirectory'] || !file_exists($pa_options['logDirectory'])) {
if (!($pa_options['logDirectory'] = $o_config->get('batch_metadata_import_log_directory'))) {
$pa_options['logDirectory'] = ".";
}
}
if (!is_writeable($pa_options['logDirectory'])) {
$pa_options['logDirectory'] = caGetTempDirPath();
}
$o_log = new KLogger($pa_options['logDirectory'], $pa_options['logLevel']);
$vb_show_cli_progress_bar = isset($pa_options['showCLIProgressBar']) && $pa_options['showCLIProgressBar'] ? true : false;
$o_progress = caGetOption('progressBar', $pa_options, new ProgressBar('WebUI'));
if ($vb_show_cli_progress_bar) {
$o_progress->setMode('CLI');
$o_progress->set('outputToTerminal', true);
}
if ($vb_use_ncurses = isset($pa_options['useNcurses']) && $pa_options['useNcurses'] ? true : false) {
$vb_use_ncurses = caCLIUseNcurses();
}
$vn_error_window_height = null;
$vn_max_x = $vn_max_y = null;
if ($vb_use_ncurses) {
ncurses_init();
$r_screen = ncurses_newwin(0, 0, 0, 0);
ncurses_border(0, 0, 0, 0, 0, 0, 0, 0);
ncurses_getmaxyx($r_screen, $vn_max_y, $vn_max_x);
$vn_error_window_height = $vn_max_y - 8;
$r_errors = ncurses_newwin($vn_error_window_height, $vn_max_x - 4, 4, 2);
ncurses_wborder($r_errors, 0, 0, 0, 0, 0, 0, 0, 0);
ncurses_wattron($r_errors, NCURSES_A_REVERSE);
ncurses_mvwaddstr($r_errors, 0, 1, _t(" Recent errors "));
ncurses_wattroff($r_errors, NCURSES_A_REVERSE);
$r_progress = ncurses_newwin(3, $vn_max_x - 4, $vn_max_y - 4, 2);
ncurses_wborder($r_progress, 0, 0, 0, 0, 0, 0, 0, 0);
ncurses_wattron($r_progress, NCURSES_A_REVERSE);
ncurses_mvwaddstr($r_progress, 0, 1, _t(" Progress "));
ncurses_wattroff($r_progress, NCURSES_A_REVERSE);
$r_status = ncurses_newwin(3, $vn_max_x - 4, 1, 2);
ncurses_wborder($r_status, 0, 0, 0, 0, 0, 0, 0, 0);
ncurses_wattron($r_status, NCURSES_A_REVERSE);
ncurses_mvwaddstr($r_status, 0, 1, _t(" Import status "));
ncurses_wattroff($r_status, NCURSES_A_REVERSE);
ncurses_refresh(0);
ncurses_wrefresh($r_progress);
ncurses_wrefresh($r_errors);
ncurses_wrefresh($r_status);
}
$o_log->logInfo(_t('Started import of %1 using mapping %2', $ps_source, $t_mapping->get("importer_code")));
$t = new Timer();
$vn_start_time = time();
$va_log_import_error_opts = array('startTime' => $vn_start_time, 'window' => $r_errors, 'log' => $o_log, 'request' => $po_request, 'progressCallback' => isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback']) ? $ps_callback : null, 'reportCallback' => isset($pa_options['reportCallback']) && ($ps_callback = $pa_options['reportCallback']) ? $ps_callback : null);
global $g_ui_locale_id;
// constant locale set by index.php for web requests
$vn_locale_id = isset($pa_options['locale_id']) && (int) $pa_options['locale_id'] ? (int) $pa_options['locale_id'] : $g_ui_locale_id;
$o_dm = $t_mapping->getAppDatamodel();
$o_progress->start(_t('Reading %1', $ps_source), array('window' => $r_progress));
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $pn_file_number, $pn_number_of_files, $ps_source, 0, 100, _t('Reading %1', $ps_source), time() - $vn_start_time, memory_get_usage(true), 0, ca_data_importers::$s_num_import_errors);
}
// Open file
$ps_format = isset($pa_options['format']) && $pa_options['format'] ? $pa_options['format'] : null;
if (!($o_reader = $t_mapping->getDataReader($ps_source, $ps_format))) {
ca_data_importers::logImportError(_t("Could not open source %1 (format=%2)", $ps_source, $ps_format), $va_log_import_error_opts);
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
if (!$o_reader->read($ps_source, array('basePath' => $t_mapping->getSetting('basePath')))) {
ca_data_importers::logImportError(_t("Could not read source %1 (format=%2)", $ps_source, $ps_format), $va_log_import_error_opts);
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
$o_log->logDebug(_t('Finished reading input source at %1 seconds', $t->getTime(4)));
$vn_num_items = $o_reader->numRows();
$o_progress->setTotal($vn_num_items);
$o_progress->start(_t('Importing from %1', $ps_source), array('window' => $r_progress));
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $pn_file_number, $pn_number_of_files, $ps_source, 0, $vn_num_items, _t('Importing from %1', $ps_source), time() - $vn_start_time, memory_get_usage(true), 0, ca_data_importers::$s_num_import_errors);
}
// What are we importing?
$vn_table_num = $t_mapping->get('table_num');
if (!($t_subject = $o_dm->getInstanceByTableNum($vn_table_num))) {
// invalid table
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
$t_subject->setTransaction($o_trans);
$vs_subject_table_name = $t_subject->tableName();
$t_label = $t_subject->getLabelTableInstance();
$t_label->setTransaction($o_trans);
$vs_label_display_fld = $t_subject->getLabelDisplayField();
$vs_subject_table = $t_subject->tableName();
$vs_type_id_fld = $t_subject->getTypeFieldName();
$vs_idno_fld = $t_subject->getProperty('ID_NUMBERING_ID_FIELD');
// get mapping rules
$va_mapping_rules = $t_mapping->getRules();
// get mapping groups
$va_mapping_groups = $t_mapping->getGroups();
$va_mapping_items = $t_mapping->getItems();
//
// Mapping-level settings
//
$vs_type_mapping_setting = $t_mapping->getSetting('type');
$vn_num_initial_rows_to_skip = $t_mapping->getSetting('numInitialRowsToSkip');
if (!in_array($vs_import_error_policy = $t_mapping->getSetting('errorPolicy'), array('ignore', 'stop'))) {
$vs_import_error_policy = 'ignore';
}
if (!in_array($vs_existing_record_policy = $t_mapping->getSetting('existingRecordPolicy'), array('none', 'skip_on_idno', 'skip_on_preferred_labels', 'merge_on_idno', 'merge_on_preferred_labels', 'merge_on_idno_and_preferred_labels', 'merge_on_idno_with_replace', 'merge_on_preferred_labels_with_replace', 'merge_on_idno_and_preferred_labels_with_replace', 'overwrite_on_idno', 'overwrite_on_preferred_labels', 'overwrite_on_idno_and_preferred_labels'))) {
$vs_existing_record_policy = 'none';
}
// Analyze mapping for figure out where type, idno, preferred label and other mandatory fields are coming from
$vn_type_id_mapping_item_id = $vn_idno_mapping_item_id = null;
$va_preferred_label_mapping_ids = array();
$va_mandatory_field_mapping_ids = array();
$va_mandatory_fields = $t_subject->getMandatoryFields();
foreach ($va_mapping_items as $vn_item_id => $va_item) {
$vs_destination = $va_item['destination'];
if (sizeof($va_dest_tmp = explode(".", $vs_destination)) >= 2) {
if ($va_dest_tmp[0] == $vs_subject_table && $va_dest_tmp[1] == 'preferred_labels') {
if (isset($va_dest_tmp[2])) {
$va_preferred_label_mapping_ids[$vn_item_id] = $va_dest_tmp[2];
} else {
$va_preferred_label_mapping_ids[$vn_item_id] = $vs_label_display_fld;
}
continue;
}
}
switch ($vs_destination) {
case 'representation_id':
if ($vs_subject_table == 'ca_representation_annotations') {
$vn_type_id_mapping_item_id = $vn_item_id;
}
break;
case "{$vs_subject_table}.{$vs_type_id_fld}":
$vn_type_id_mapping_item_id = $vn_item_id;
break;
case "{$vs_subject_table}.{$vs_idno_fld}":
$vn_idno_mapping_item_id = $vn_item_id;
break;
}
foreach ($va_mandatory_fields as $vs_mandatory_field) {
if ($vs_mandatory_field == $vs_type_id_fld) {
continue;
}
// type is handled separately
if ($vs_destination == "{$vs_subject_table}.{$vs_mandatory_field}") {
$va_mandatory_field_mapping_ids[$vs_mandatory_field] = $vn_item_id;
}
}
}
$va_items_by_group = array();
foreach ($va_mapping_items as $vn_item_id => $va_item) {
$va_items_by_group[$va_item['group_id']][$va_item['item_id']] = $va_item;
}
$o_log->logDebug(_t('Finished analyzing mapping at %1 seconds', $t->getTime(4)));
//
// Set up environment
//
$va_environment = caGetOption('environment', $pa_options, array(), array('castTo' => 'array'));
if (is_array($va_environment_config = $t_mapping->getEnvironment())) {
foreach ($va_environment_config as $vn_i => $va_environment_item) {
$va_env_tmp = explode("|", $va_environment_item['value']);
if (!($o_env_reader = $t_mapping->getDataReader($ps_source, $ps_format))) {
break;
}
if (!$o_env_reader->read($ps_source, array('basePath' => ''))) {
break;
}
$o_env_reader->nextRow();
switch (sizeof($va_env_tmp)) {
case 1:
$vs_env_value = $o_env_reader->get($va_environment_item['value'], array('returnAsArray' => false));
break;
case 2:
$vn_seek = (int) $va_env_tmp[0];
$o_reader->seek($vn_seek > 0 ? $vn_seek - 1 : $vn_seek);
$o_env_reader->nextRow();
$vs_env_value = $o_env_reader->get($va_env_tmp[1], array('returnAsArray' => false));
$o_reader->seek(0);
break;
default:
$vs_env_value = $va_environment_item['value'];
break;
}
$va_environment[$va_environment_item['name']] = $vs_env_value;
}
}
//
// Run through rows
//
$vn_row = 0;
ca_data_importers::$s_num_records_processed = 0;
while ($o_reader->nextRow()) {
$va_mandatory_field_values = array();
$vs_preferred_label_for_log = null;
if ($vn_row < $vn_num_initial_rows_to_skip) {
// skip over initial header rows
$vn_row++;
continue;
}
$vn_row++;
$t->startTimer();
$o_log->logDebug(_t('Started reading row %1 at %2 seconds', $vn_row, $t->getTime(4)));
$t_subject = $o_dm->getInstanceByTableNum($vn_table_num);
if ($o_trans) {
$t_subject->setTransaction($o_trans);
}
$t_subject->setMode(ACCESS_WRITE);
// Update status display
if ($vb_use_ncurses && ca_data_importers::$s_num_records_processed) {
ncurses_mvwaddstr($r_status, 1, 2, _t("Items processed/skipped: %1/%2", ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_records_skipped) . str_repeat(" ", 5) . _t("Errors: %1 (%2)", ca_data_importers::$s_num_import_errors, sprintf("%3.1f", ca_data_importers::$s_num_import_errors / ca_data_importers::$s_num_records_processed * 100) . "%") . str_repeat(" ", 6) . _t("Mapping: %1", $ps_mapping) . str_repeat(" ", 5) . _t("Source: %1", $ps_source) . str_repeat(" ", 5) . date("Y-m-d H:i:s") . str_repeat(" ", 5));
ncurses_refresh(0);
ncurses_wrefresh($r_status);
}
//
// Get data for current row
//
$va_row = array_merge($o_reader->getRow(), $va_environment);
//
// Apply rules
//
foreach ($va_mapping_rules as $va_rule) {
if (!isset($va_rule['trigger']) || !$va_rule['trigger']) {
continue;
}
if (!isset($va_rule['actions']) || !is_array($va_rule['actions']) || !sizeof($va_rule['actions'])) {
continue;
}
$vm_ret = ExpressionParser::evaluate($va_rule['trigger'], $va_row);
if (!ExpressionParser::hadError() && (bool) $vm_ret) {
foreach ($va_rule['actions'] as $va_action) {
if (!is_array($va_action) && strtolower($va_action) == 'skip') {
$va_action = array('action' => 'skip');
}
switch ($vs_action_code = strtolower($va_action['action'])) {
case 'set':
$va_row[$va_action['target']] = $va_action['value'];
// TODO: transform value using mapping rules?
break;
case 'skip':
default:
if ($vs_action_code != 'skip') {
$o_log->logInfo(_t('Row was skipped using rule "%1" with default action because an invalid action ("%2") was specified', $va_rule['trigger'], $vs_action_code));
} else {
$o_log->logDebug(_t('Row was skipped using rule "%1" with action "%2"', $va_rule['trigger'], $vs_action_code));
}
continue 4;
break;
}
}
}
}
//
// Perform mapping and insert
//
// Get minimal info for imported row (type_id, idno, label)
// Get type
if ($vn_type_id_mapping_item_id) {
// Type is specified in row
$vs_type = ca_data_importers::getValueFromSource($va_mapping_items[$vn_type_id_mapping_item_id], $o_reader, array('environment' => $va_environment));
} else {
// Type is constant for all rows
$vs_type = $vs_type_mapping_setting;
}
// Get idno
$vs_idno = null;
if ($vn_idno_mapping_item_id) {
// idno is specified in row
$vs_idno = ca_data_importers::getValueFromSource($va_mapping_items[$vn_idno_mapping_item_id], $o_reader, array('environment' => $va_environment));
if (isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['default']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['default']) && !strlen($vs_idno)) {
$vs_idno = $va_mapping_items[$vn_idno_mapping_item_id]['settings']['default'];
}
if (!is_array($vs_idno) && $vs_idno[0] == '^' && preg_match("!^\\^[^ ]+\$!", $vs_idno)) {
// Parse placeholder when it's at the beginning of the value
if (!is_null($vm_parsed_val = BaseRefinery::parsePlaceholder($vs_idno, $va_row, $va_item, null, array('reader' => $o_reader, 'returnAsString' => true)))) {
$vs_idno = $vm_parsed_val;
}
}
// Apply prefix/suffix *AFTER* setting default
if ($vs_idno && isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix'])) {
$vs_idno = $va_mapping_items[$vn_idno_mapping_item_id]['settings']['prefix'] . $vs_idno;
}
if ($vs_idno && isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix'])) {
$vs_idno .= $va_mapping_items[$vn_idno_mapping_item_id]['settings']['suffix'];
}
if (isset($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate']) && strlen($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate'])) {
$vs_idno = caProcessTemplate($va_mapping_items[$vn_idno_mapping_item_id]['settings']['formatWithTemplate'], $va_row);
}
} else {
$vs_idno = "%";
}
$vb_idno_is_template = (bool) preg_match('![%]+!', $vs_idno);
// get preferred labels
$va_pref_label_values = array();
foreach ($va_preferred_label_mapping_ids as $vn_preferred_label_mapping_id => $vs_preferred_label_mapping_fld) {
$vs_label_val = ca_data_importers::getValueFromSource($va_mapping_items[$vn_preferred_label_mapping_id], $o_reader, array('environment' => $va_environment));
// If a template is specified format the label value with it so merge-on-preferred_label doesn't fail
if (isset($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate']) && strlen($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate'])) {
$vs_label_val = caProcessTemplate($va_mapping_items[$vn_preferred_label_mapping_id]['settings']['formatWithTemplate'], $va_row);
}
$va_pref_label_values[$vs_preferred_label_mapping_fld] = $vs_label_val;
}
$vs_display_label = isset($va_pref_label_values[$vs_label_display_fld]) ? $va_pref_label_values[$vs_label_display_fld] : $vs_idno;
//
// Look for existing record?
//
if (is_array($pa_force_import_for_primary_keys) && sizeof($pa_force_import_for_primary_keys) > 0) {
$vn_id = array_shift($pa_force_import_for_primary_keys);
if (!$t_subject->load($vn_id)) {
$o_log->logInfo(_t('[%1] Skipped import because of forced primary key \'%1\' does not exist', $vn_id));
ca_data_importers::$s_num_records_skipped++;
continue;
// skip because primary key does not exist
}
} elseif ($vs_existing_record_policy != 'none') {
switch ($vs_existing_record_policy) {
case 'skip_on_idno':
if (!$vb_idno_is_template) {
$va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, $t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno, 'deleted' => 0), array('returnAs' => 'ids')));
if (is_array($va_ids) && sizeof($va_ids) > 0) {
$o_log->logInfo(_t('[%1] Skipped import because of existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy));
ca_data_importers::$s_num_records_skipped++;
continue 2;
// skip because idno matched
}
}
break;
case 'skip_on_preferred_labels':
$va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, 'preferred_labels' => $va_pref_label_values, 'deleted' => 0), array('returnAs' => 'ids')));
if (is_array($va_ids) && sizeof($va_ids) > 0) {
$o_log->logInfo(_t('[%1] Skipped import because of existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy));
ca_data_importers::$s_num_records_skipped++;
continue 2;
// skip because label matched
}
break;
case 'merge_on_idno_and_preferred_labels':
case 'merge_on_idno':
case 'merge_on_idno_and_preferred_labels_with_replace':
case 'merge_on_idno_with_replace':
if (!$vb_idno_is_template) {
$va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, $t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno, 'deleted' => 0), array('returnAs' => 'ids')));
if (is_array($va_ids) && sizeof($va_ids) > 0) {
$t_subject->load($va_ids[0]);
$o_log->logInfo(_t('[%1] Merged with existing record matched on identifer by policy %2', $vs_idno, $vs_existing_record_policy));
break;
}
}
if (in_array($vs_existing_record_policy, array('merge_on_idno', 'merge_on_idno_with_replace'))) {
break;
}
// fall through if merge_on_idno_and_preferred_labels
// fall through if merge_on_idno_and_preferred_labels
case 'merge_on_preferred_labels':
case 'merge_on_preferred_labels_with_replace':
$va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, 'preferred_labels' => $va_pref_label_values, 'deleted' => 0), array('returnAs' => 'ids')));
if (is_array($va_ids) && sizeof($va_ids) > 0) {
$t_subject->load($va_ids[0]);
$o_log->logInfo(_t('[%1] Merged with existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy));
}
break;
case 'overwrite_on_idno_and_preferred_labels':
case 'overwrite_on_idno':
if (!$vb_idno_is_template && $vs_idno) {
$va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, $t_subject->getProperty('ID_NUMBERING_ID_FIELD') => $vs_idno, 'deleted' => 0), array('returnAs' => 'ids')));
if (is_array($va_ids) && sizeof($va_ids) > 0) {
$t_subject->load($va_ids[0]);
$t_subject->setMode(ACCESS_WRITE);
$t_subject->delete(true, array('hard' => true));
if ($t_subject->numErrors()) {
ca_data_importers::logImportError(_t('[%1] Could not delete existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy));
// Don't stop?
} else {
$o_log->logInfo(_t('[%1] Overwrote existing record matched on identifier by policy %2', $vs_idno, $vs_existing_record_policy));
$t_subject->clear();
break;
}
}
}
if ($vs_existing_record_policy == 'overwrite_on_idno') {
break;
}
// fall through if overwrite_on_idno_and_preferred_labels
// fall through if overwrite_on_idno_and_preferred_labels
case 'overwrite_on_preferred_labels':
$va_ids = call_user_func_array($t_subject->tableName() . "::find", array(array('type_id' => $vs_type, 'preferred_labels' => $va_pref_label_values, 'deleted' => 0), array('returnAs' => 'ids')));
if (is_array($va_ids) && sizeof($va_ids) > 0) {
$t_subject->load($va_ids[0]);
$t_subject->setMode(ACCESS_WRITE);
$t_subject->delete(true, array('hard' => true));
if ($t_subject->numErrors()) {
ca_data_importers::logImportError(_t('[%1] Could not delete existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy));
// Don't stop?
} else {
$o_log->logInfo(_t('[%1] Overwrote existing record matched on label by policy %2', $vs_idno, $vs_existing_record_policy));
break;
}
$t_subject->clear();
}
break;
}
}
$o_progress->next(_t("Importing %1", $vs_idno), array('window' => $r_progress));
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $pn_file_number, $pn_number_of_files, $ps_source, ca_data_importers::$s_num_records_processed, $vn_num_items, _t("[%3/%4] Processing %1 (%2)", caTruncateStringWithEllipsis($vs_display_label, 50), $vs_idno, ca_data_importers::$s_num_records_processed, $vn_num_items), time() - $vn_start_time, memory_get_usage(true), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_import_errors);
}
$vb_output_subject_preferred_label = false;
$va_content_tree = array();
foreach ($va_items_by_group as $vn_group_id => $va_items) {
$va_group = $va_mapping_groups[$vn_group_id];
$vs_group_destination = $va_group['destination'];
$va_group_tmp = explode(".", $vs_group_destination);
if (sizeof($va_items) < 2 && sizeof($va_group_tmp) > 2) {
array_pop($va_group_tmp);
}
$vs_target_table = $va_group_tmp[0];
if (!($t_target = $o_dm->getInstanceByTableName($vs_target_table, true))) {
// Invalid target table
$o_log->logWarn(_t('[%1] Skipped group %2 because target %3 is invalid', $vs_idno, $vn_group_id, $vs_target_table));
continue;
}
if ($o_trans) {
$t_target->setTransaction($o_trans);
}
$va_group_buf = array();
foreach ($va_items as $vn_item_id => $va_item) {
if ($vb_use_as_single_value = caGetOption('useAsSingleValue', $va_item['settings'], false)) {
// Force repeating values to be imported as a single value
$va_vals = array(ca_data_importers::getValueFromSource($va_item, $o_reader, array('delimiter' => caGetOption('delimiter', $va_item['settings'], ''), 'returnAsArray' => false)));
} else {
$va_vals = ca_data_importers::getValueFromSource($va_item, $o_reader, array('returnAsArray' => true, 'environment' => $va_environment));
}
if (!sizeof($va_vals)) {
$va_vals = array(0 => null);
}
// consider missing values equivalent to blanks
// Do value conversions
foreach ($va_vals as $vn_i => $vm_val) {
if (isset($va_item['settings']['default']) && strlen($va_item['settings']['default']) && !strlen($vm_val)) {
$vm_val = $va_item['settings']['default'];
}
// Apply prefix/suffix *AFTER* setting default
if ($vm_val && isset($va_item['settings']['prefix']) && strlen($va_item['settings']['prefix'])) {
$vm_val = $va_item['settings']['prefix'] . $vm_val;
}
if ($vm_val && isset($va_item['settings']['suffix']) && strlen($va_item['settings']['suffix'])) {
$vm_val .= $va_item['settings']['suffix'];
}
if (!is_array($vm_val) && $vm_val[0] == '^' && preg_match("!^\\^[^ ]+\$!", $vm_val)) {
// Parse placeholder
if (!is_null($vm_parsed_val = BaseRefinery::parsePlaceholder($vm_val, $va_row, $va_item, $vn_i, array('reader' => $o_reader, 'returnAsString' => true)))) {
$vm_val = $vm_parsed_val;
}
}
if (isset($va_item['settings']['formatWithTemplate']) && strlen($va_item['settings']['formatWithTemplate'])) {
$vm_val = caProcessTemplate($va_item['settings']['formatWithTemplate'], array_replace($va_row, array((string) $va_item['source'] => ca_data_importers::replaceValue($vm_val, $va_item))), array('getFrom' => $o_reader));
}
if (isset($va_item['settings']['applyRegularExpressions']) && is_array($va_item['settings']['applyRegularExpressions'])) {
if (is_array($va_item['settings']['applyRegularExpressions'])) {
foreach ($va_item['settings']['applyRegularExpressions'] as $vn_regex_index => $va_regex) {
if (!strlen($va_regex['match'])) {
continue;
}
$vm_val = preg_replace("!" . str_replace("!", "\\!", $va_regex['match']) . "!" . (isset($va_regex['caseSensitive']) && (bool) $va_regex['caseSensitive'] ? '' : 'i'), $va_regex['replaceWith'], $vm_val);
}
}
}
$va_vals[$vn_i] = $vm_val;
if ($o_reader->valuesCanRepeat()) {
$va_row[$va_item['source']][$vn_i] = $va_row[mb_strtolower($va_item['source'])][$vn_i] = $vm_val;
} else {
$va_row[$va_item['source']] = $va_row[mb_strtolower($va_item['source'])] = $vm_val;
}
}
// Process each value
$vn_c = -1;
foreach ($va_vals as $vn_i => $vm_val) {
$vn_c++;
if (isset($va_item['settings']['convertNewlinesToHTML']) && (bool) $va_item['settings']['convertNewlinesToHTML'] && is_string($vm_val)) {
$vm_val = nl2br($vm_val);
}
// Get location in content tree for addition of new content
$va_item_dest = explode(".", $va_item['destination']);
$vs_item_terminal = $va_item_dest[sizeof($va_item_dest) - 1];
if (isset($va_item['settings']['restrictToTypes']) && is_array($va_item['settings']['restrictToTypes']) && !in_array($vs_type, $va_item['settings']['restrictToTypes'])) {
$o_log->logInfo(_t('[%1] Skipped row %2 because of type restriction', $vs_idno, $vn_row));
continue 4;
}
if (isset($va_item['settings']['skipRowIfEmpty']) && (bool) $va_item['settings']['skipRowIfEmpty'] && !strlen($vm_val)) {
$o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 is empty', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id));
continue 4;
}
if (isset($va_item['settings']['skipRowIfValue']) && is_array($va_item['settings']['skipRowIfValue']) && strlen($vm_val) && in_array($vm_val, $va_item['settings']['skipRowIfValue'])) {
$o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 matches value %5', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id));
continue 4;
}
if (isset($va_item['settings']['skipRowIfNotValue']) && is_array($va_item['settings']['skipRowIfNotValue']) && strlen($vm_val) && !in_array($vm_val, $va_item['settings']['skipRowIfNotValue'])) {
$o_log->logInfo(_t('[%1] Skipped row %2 because value for %3 in group %4 is not in list of values', $vs_idno, $vn_row, $vs_item_terminal, $vn_group_id, $vm_val));
continue 4;
}
if (isset($va_item['settings']['skipGroupIfEmpty']) && (bool) $va_item['settings']['skipGroupIfEmpty'] && !strlen($vm_val)) {
$o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 is empty', $vs_idno, $vn_group_id, $vs_item_terminal));
continue 3;
}
if (isset($va_item['settings']['skipGroupIfExpression']) && strlen(trim($va_item['settings']['skipGroupIfExpression']))) {
if ($vm_ret = ExpressionParser::evaluate($va_item['settings']['skipGroupIfExpression'], $va_row)) {
$o_log->logInfo(_t('[%1] Skipped group %2 because expression %3 is true', $vs_idno, $vn_group_id, $va_item['settings']['skipGroupIfExpression']));
continue 3;
}
}
if (isset($va_item['settings']['skipGroupIfValue']) && is_array($va_item['settings']['skipGroupIfValue']) && strlen($vm_val) && in_array($vm_val, $va_item['settings']['skipGroupIfValue'])) {
$o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 matches value %4', $vs_idno, $vn_group_id, $vs_item_terminal, $vm_val));
continue 3;
}
if (isset($va_item['settings']['skipGroupIfNotValue']) && is_array($va_item['settings']['skipGroupIfNotValue']) && strlen($vm_val) && !in_array($vm_val, $va_item['settings']['skipGroupIfNotValue'])) {
$o_log->logInfo(_t('[%1] Skipped group %2 because value for %3 matches is not in list of values', $vs_idno, $vn_group_id, $vs_item_terminal));
continue 3;
}
if (isset($va_item['settings']['skipIfExpression']) && strlen(trim($va_item['settings']['skipIfExpression']))) {
if ($vm_ret = ExpressionParser::evaluate($va_item['settings']['skipIfExpression'], $va_row)) {
$o_log->logInfo(_t('[%1] Skipped mapping because expression %2 is true', $vs_idno, $va_item['settings']['skipIfExpression']));
continue 2;
}
}
if (isset($va_item['settings']['skipIfEmpty']) && (bool) $va_item['settings']['skipIfEmpty'] && !strlen($vm_val)) {
$o_log->logInfo(_t('[%1] Skipped mapping because value for %2 is empty', $vs_idno, $vs_item_terminal));
continue 2;
}
if ($vn_type_id_mapping_item_id && $vn_item_id == $vn_type_id_mapping_item_id) {
continue;
}
if ($vn_idno_mapping_item_id && $vn_item_id == $vn_idno_mapping_item_id) {
continue;
}
if (is_null($vm_val)) {
continue;
}
// Get mapping error policy
$vb_item_error_policy_is_default = false;
if (!isset($va_item['settings']['errorPolicy']) || !in_array($vs_item_error_policy = $va_item['settings']['errorPolicy'], array('ignore', 'stop'))) {
$vs_item_error_policy = 'ignore';
$vb_item_error_policy_is_default = true;
}
//
if (isset($va_item['settings']['relationshipType']) && strlen($vs_rel_type = $va_item['settings']['relationshipType']) && $vs_target_table != $vs_subject_table) {
$va_group_buf[$vn_c]['_relationship_type'] = $vs_rel_type;
}
// Is it a constant value?
if (preg_match("!^_CONSTANT_:[\\d]+:(.*)!", $va_item['source'], $va_matches)) {
$va_group_buf[$vn_c][$vs_item_terminal] = $va_matches[1];
// Set it and go onto the next item
if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) {
$va_mandatory_field_values[$vs_k] = $vm_val;
}
continue;
}
// Perform refinery call (if required) per value
if (isset($va_item['settings']['refineries']) && is_array($va_item['settings']['refineries'])) {
foreach ($va_item['settings']['refineries'] as $vs_refinery) {
if (!$vs_refinery) {
continue;
}
if ($o_refinery = RefineryManager::getRefineryInstance($vs_refinery)) {
$va_refined_values = $o_refinery->refine($va_content_tree, $va_group, $va_item, $va_row, array('mapping' => $t_mapping, 'source' => $ps_source, 'subject' => $t_subject, 'locale_id' => $vn_locale_id, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'reader' => $o_reader, 'valueIndex' => $vn_i));
if (!$va_refined_values || is_array($va_refined_values) && !sizeof($va_refined_values)) {
continue 2;
}
if ($o_refinery->returnsMultipleValues()) {
foreach ($va_refined_values as $va_refined_value) {
$va_refined_value['_errorPolicy'] = $vs_item_error_policy;
if (!is_array($va_group_buf[$vn_c])) {
$va_group_buf[$vn_c] = array();
}
$va_group_buf[$vn_c] = array_merge($va_group_buf[$vn_c], $va_refined_value);
$vn_c++;
}
} else {
$va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy;
$va_group_buf[$vn_c][$vs_item_terminal] = $va_refined_values;
$vn_c++;
}
if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) {
$va_mandatory_field_values[$vs_k] = $vm_val;
}
continue 2;
} else {
ca_data_importers::logImportError(_t('[%1] Invalid refinery %2 specified', $vs_idno, $vs_refinery));
}
}
}
if ($vs_target_table == $vs_subject_table_name && ($vs_k = array_search($vn_item_id, $va_mandatory_field_mapping_ids)) !== false) {
$va_mandatory_field_values[$vs_k] = $vm_val;
}
$vn_max_length = !is_array($vm_val) && isset($va_item['settings']['maxLength']) && (int) $va_item['settings']['maxLength'] ? (int) $va_item['settings']['maxLength'] : null;
if (isset($va_item['settings']['delimiter']) && $va_item['settings']['delimiter']) {
if (!is_array($va_item['settings']['delimiter'])) {
$va_item['settings']['delimiter'] = array($va_item['settings']['delimiter']);
}
if (sizeof($va_item['settings']['delimiter'])) {
foreach ($va_item['settings']['delimiter'] as $vn_index => $vs_delim) {
$va_item['settings']['delimiter'][$vn_index] = preg_quote($vs_delim, "!");
}
$va_val_list = preg_split("!(" . join("|", $va_item['settings']['delimiter']) . ")!", $vm_val);
// Add delimited values
foreach ($va_val_list as $vs_list_val) {
$vs_list_val = trim(ca_data_importers::replaceValue($vs_list_val, $va_item));
if ($vn_max_length && mb_strlen($vs_list_val) > $vn_max_length) {
$vs_list_val = mb_substr($vs_list_val, 0, $vn_max_length);
}
$va_group_buf[$vn_c] = array($vs_item_terminal => $vs_list_val, '_errorPolicy' => $vs_item_error_policy);
$vn_c++;
}
$vn_row++;
continue;
// Don't add "regular" value below
}
}
if ($vn_max_length && mb_strlen($vm_val) > $vn_max_length) {
$vm_val = mb_substr($vm_val, 0, $vn_max_length);
}
switch ($vs_item_terminal) {
case 'preferred_labels':
case 'nonpreferred_labels':
if ($t_instance = $o_dm->getInstanceByTableName($vs_target_table, true)) {
$va_group_buf[$vn_c][$t_instance->getLabelDisplayField()] = $vm_val;
}
if ($o_trans) {
$t_instance->setTransaction($o_trans);
}
if (!$vb_item_error_policy_is_default || !isset($va_group_buf[$vn_c]['_errorPolicy'])) {
if (is_array($va_group_buf[$vn_c])) {
$va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy;
}
}
if ($vs_item_terminal == 'preferred_labels') {
$vs_preferred_label_for_log = $vm_val;
}
break;
default:
$va_group_buf[$vn_c][$vs_item_terminal] = $vm_val;
if (!$vb_item_error_policy_is_default || !isset($va_group_buf[$vn_c]['_errorPolicy'])) {
if (is_array($va_group_buf[$vn_c])) {
$va_group_buf[$vn_c]['_errorPolicy'] = $vs_item_error_policy;
}
}
break;
}
}
// end foreach($va_vals as $vm_val)
}
foreach ($va_group_buf as $vn_group_index => $va_group_data) {
$va_ptr =& $va_content_tree;
foreach ($va_group_tmp as $vs_tmp) {
if (!is_array($va_ptr[$vs_tmp])) {
$va_ptr[$vs_tmp] = array();
}
$va_ptr =& $va_ptr[$vs_tmp];
if ($vs_tmp == $vs_target_table) {
// add numeric index after table to ensure repeat values don't overwrite each other
$va_parent =& $va_ptr;
$va_ptr[] = array();
$va_ptr =& $va_ptr[sizeof($va_ptr) - 1];
}
}
$va_ptr = $va_group_data;
}
}
//
// Process out self-relationships
//
if (is_array($va_content_tree[$vs_subject_table])) {
$va_self_related_content = array();
foreach ($va_content_tree[$vs_subject_table] as $vn_i => $va_element_data) {
if (isset($va_element_data['_relationship_type'])) {
$va_self_related_content[] = $va_element_data;
unset($va_content_tree[$vs_subject_table][$vn_i]);
}
}
if (sizeof($va_self_related_content) > 0) {
$va_content_tree["related.{$vs_subject_table}"] = $va_self_related_content;
}
}
$vn_row++;
$o_log->logDebug(_t('Finished building content tree for %1 at %2 seconds', $vs_idno, $t->getTime(4)));
$o_log->logDebug(_t("Content tree is\n%1", print_R($va_content_tree, true)));
//
// Process data in subject record
//
//print_r($va_content_tree);
//die("END\n\n");
//continue;
$opa_app_plugin_manager->hookDataImportContentTree(array('mapping' => $t_mapping, 'content_tree' => &$va_content_tree, 'idno' => &$vs_idno, 'transaction' => &$o_trans, 'log' => &$o_log, 'reader' => $o_reader, 'environment' => $va_environment, 'importEvent' => $o_event, 'importEventSource' => $vn_row));
//print_r($va_content_tree);
//die("done\n");
if (!sizeof($va_content_tree) && !str_replace("%", "", $vs_idno)) {
continue;
}
if (!$t_subject->getPrimaryKey()) {
$o_event->beginItem($vn_row, $t_subject->tableNum(), 'I');
$t_subject->setMode(ACCESS_WRITE);
$t_subject->set($vs_type_id_fld, $vs_type);
if ($vb_idno_is_template) {
$t_subject->setIdnoWithTemplate($vs_idno);
} else {
$t_subject->set($vs_idno_fld, $vs_idno, array('assumeIdnoForRepresentationID' => true, 'assumeIdnoStubForLotID' => true));
// assumeIdnoStubForLotID forces ca_objects.lot_id values to always be considered as a potential idno_stub first, before use as a ca_objects.lot_id
}
// Look for parent_id in the content tree
$vs_parent_id_fld = $t_subject->getProperty('HIERARCHY_PARENT_ID_FLD');
foreach ($va_content_tree as $vs_table_name => $va_content) {
if ($vs_table_name == $vs_subject_table) {
foreach ($va_content as $va_element_data) {
foreach ($va_element_data as $vs_element => $va_element_content) {
switch ($vs_element) {
case $vs_parent_id_fld:
if ($va_element_content[$vs_parent_id_fld]) {
$t_subject->set($vs_parent_id_fld, $va_element_content[$vs_parent_id_fld], array('treatParentIDAsIdno' => true));
}
break;
}
}
}
}
}
foreach ($va_mandatory_field_mapping_ids as $vs_mandatory_field => $vn_mandatory_mapping_item_id) {
$t_subject->set($vs_mandatory_field, $va_mandatory_field_values[$vs_mandatory_field], array('assumeIdnoStubForLotID' => true));
}
$t_subject->insert();
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not insert new record"), array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
continue;
}
$o_log->logDebug(_t('Created idno %1 at %2 seconds', $vs_idno, $t->getTime(4)));
} else {
$o_event->beginItem($vn_row, $t_subject->tableNum(), 'U');
// update
$t_subject->setMode(ACCESS_WRITE);
if ($vb_idno_is_template) {
$t_subject->setIdnoWithTemplate($vs_idno);
} else {
$t_subject->set($vs_idno_fld, $vs_idno, array('assumeIdnoStubForLotID' => true));
// assumeIdnoStubForLotID forces ca_objects.lot_id values to always be considered as a potential idno_stub first, before use as a ca_objects.lot_di
}
$t_subject->update();
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not update matched record"), array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
continue;
}
$t_subject->clearErrors();
if (sizeof($va_preferred_label_mapping_ids) && $t_subject->getPreferredLabelCount() > 0) {
$t_subject->removeAllLabels(__CA_LABEL_TYPE_PREFERRED__);
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could not update remove preferred labels from matched record"), array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
return false;
}
}
}
$o_log->logDebug(_t('Updated idno %1 at %2 seconds', $vs_idno, $t->getTime(4)));
}
if ($vs_idno_fld && ($o_idno = $t_subject->getIDNoPlugInInstance())) {
$va_values = $o_idno->htmlFormValuesAsArray($vs_idno_fld, $t_subject->get($vs_idno_fld));
if (!is_array($va_values)) {
$va_values = array($va_values);
}
if (($vs_proc_idno = join($o_idno->getSeparator(), $va_values)) && $vs_proc_idno != $vs_idno) {
$t_subject->set($vs_idno_fld, $vs_proc_idno);
$t_subject->update();
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("Could update idno"), array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
continue;
}
}
}
$va_elements_set_for_this_record = array();
foreach ($va_content_tree as $vs_table_name => $va_content) {
if ($vs_table_name == $vs_subject_table) {
foreach ($va_content as $vn_i => $va_element_data) {
foreach ($va_element_data as $vs_element => $va_element_content) {
if (is_array($va_element_content)) {
$vs_item_error_policy = $va_element_content['_errorPolicy'];
unset($va_element_content['_errorPolicy']);
} else {
$vs_item_error_policy = null;
}
$t_subject->clearErrors();
$t_subject->setMode(ACCESS_WRITE);
switch ($vs_element) {
case 'preferred_labels':
$t_subject->addLabel($va_element_content, $vn_locale_id, isset($va_element_content['type_id']) ? $va_element_content['type_id'] : null, true);
if ($t_subject->numErrors() == 0) {
$vb_output_subject_preferred_label = true;
}
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add preferred label to %2. Record was deleted because no preferred label could be applied: ", $vs_idno, $t_subject->tableName()), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
$t_subject->delete(true, array('hard' => false));
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy %1', $vs_import_error_policy));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
continue 5;
}
break;
case 'nonpreferred_labels':
$t_subject->addLabel($va_element_content, $vn_locale_id, isset($va_element_content['type_id']) ? $va_element_content['type_id'] : null, false);
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add non-preferred label to %2:", $vs_idno, $t_subject->tableName()), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
break;
default:
if ($t_subject->hasField($vs_element)) {
$t_subject->set($vs_element, $va_element_content[$vs_element], array('assumeIdnoStubForLotID' => true));
$t_subject->update();
break;
}
if ($vs_subject_table == 'ca_representation_annotations' && $vs_element == 'properties') {
foreach ($va_element_content as $vs_prop => $vs_prop_val) {
$t_subject->setPropertyValue($vs_prop, $vs_prop_val);
}
break;
}
if (is_array($va_element_content)) {
$va_element_content['locale_id'] = $vn_locale_id;
}
if (!isset($va_elements_set_for_this_record[$vs_element]) && !$va_elements_set_for_this_record[$vs_element] && in_array($vs_existing_record_policy, array('merge_on_idno_with_replace', 'merge_on_preferred_labels_with_replace', 'merge_on_idno_and_preferred_labels_with_replace'))) {
$t_subject->removeAttributes($vs_element, array('force' => true));
}
$va_elements_set_for_this_record[$vs_element] = true;
$t_subject->addAttribute($va_element_content, $vs_element, null, array('showRepeatCountErrors' => true, 'alwaysTreatValueAsIdno' => true));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Failed to add value for %2; values were %3: ", $vs_idno, $vs_element, ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
$t_subject->update();
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Invalid %2; values were %3: ", $vs_idno, $vs_element, ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
break;
}
}
}
} else {
// related
$vs_table_name = preg_replace('!^related\\.!', '', $vs_table_name);
foreach ($va_content as $vn_i => $va_element_data) {
$va_match_on = caGetOption('_matchOn', $va_element_data, null);
$vb_dont_create = caGetOption('_dontCreate', $va_element_data, null);
$va_data_for_rel_table = $va_element_data;
$va_nonpreferred_labels = isset($va_data_for_rel_table['nonpreferred_labels']) ? $va_data_for_rel_table['nonpreferred_labels'] : null;
unset($va_data_for_rel_table['preferred_labels']);
unset($va_data_for_rel_table['_relationship_type']);
unset($va_data_for_rel_table['_type']);
unset($va_data_for_rel_table['_parent_id']);
unset($va_data_for_rel_table['_errorPolicy']);
unset($va_data_for_rel_table['_matchOn']);
$va_data_for_rel_table = array_merge($va_data_for_rel_table, ca_data_importers::_extractIntrinsicValues($va_data_for_rel_table, $vs_table_name));
$t_subject->clearErrors();
switch ($vs_table_name) {
case 'ca_objects':
if ($vn_rel_id = DataMigrationUtils::getObjectID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related object with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_object_lots':
$vs_idno_stub = null;
if (is_array($va_element_data['idno_stub'])) {
$vs_idno_stub = isset($va_element_data['idno_stub']['idno_stub']) ? $va_element_data['idno_stub']['idno_stub'] : '';
} else {
$vs_idno_stub = isset($va_element_data['idno_stub']) ? $va_element_data['idno_stub'] : '';
}
if ($vn_rel_id = DataMigrationUtils::getObjectLotID($vs_idno_stub, $va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno_stub']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related object lot with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_entities':
if ($vn_rel_id = DataMigrationUtils::getEntityID($va_element_data['preferred_labels'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related entity with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_places':
if ($vn_rel_id = DataMigrationUtils::getPlaceID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related place with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_collections':
if ($vn_rel_id = DataMigrationUtils::getCollectionID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, $vs_rel_type, null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related collection with relationship %2:", $vs_idno, $vs_rel_type), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_occurrences':
if ($vn_rel_id = DataMigrationUtils::getOccurrenceID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, $vs_rel_type, null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related occurrence with relationship %2:", $vs_idno, $vs_rel_type), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_storage_locations':
if ($vn_rel_id = DataMigrationUtils::getStorageLocationID($va_element_data['preferred_labels']['name'], $va_element_data['_parent_id'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related storage location with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_list_items':
$va_data_for_rel_table['is_enabled'] = 1;
$va_data_for_rel_table['preferred_labels'] = $va_element_data['preferred_labels'];
if ($vn_rel_id = DataMigrationUtils::getListItemID($va_element_data['_list'], $va_element_data['idno'] ? $va_element_data['idno'] : null, $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related list item with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_object_representations':
if ($vn_rel_id = DataMigrationUtils::getObjectRepresentationID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels, 'matchMediaFilesWithoutExtension' => true))) {
$t_subject->linkRepresentation($vn_rel_id, null, null, null, null, array('type_id' => trim($va_element_data['_relationship_type']), 'is_primary' => true));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related object representation with:", $vs_idno), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
//
// if (($vs_subject_table_name == 'ca_objects') && $va_element_data['media']['media']) {
// unset($va_data_for_rel_table['media']);
//
// foreach($va_data_for_rel_table as $vs_key => $vm_val) {
// // Attributes, including intrinsics are in two-level format, eg. idno is $va_attributes['idno']['idno']
// // but addRepresentations() expects intrinsics to be single level (eg. $va_attributes['idno']) so
// // we do some rewriting here
// if (is_array($vm_val) && isset($vm_val[$vs_key])) {
// $va_data_for_rel_table[$vs_key] = $vm_val[$vs_key];
// }
// }
//
// if (!($t_subject->addRepresentation($va_element_data['media']['media'], isset($va_element_data['_type']) ? $va_element_data['_type'] : caGetDefaultItemID('object_representation_types'), $vn_locale_id, 0, 0, true, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on)))) {
// $vs_error = join("; ", $t_subject->getErrors());
// ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
// if ($vs_item_error_policy == 'stop') {
// $o_log->logAlert(_t('Import stopped due to mapping error policy'));
// if($vb_use_ncurses) { ncurses_end(); }
// if ($o_trans) { $o_trans->rollback(); }
// return false;
// }
// }
// }
break;
case 'ca_loans':
if ($vn_rel_id = DataMigrationUtils::getLoanID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related loan with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
case 'ca_movements':
if ($vn_rel_id = DataMigrationUtils::getMovementID($va_element_data['preferred_labels']['name'], $va_element_data['_type'], $vn_locale_id, $va_data_for_rel_table, array('dontCreate' => $vb_dont_create, 'matchOn' => $va_match_on, 'log' => $o_log, 'transaction' => $o_trans, 'importEvent' => $o_event, 'importEventSource' => $vn_row, 'nonPreferredLabels' => $va_nonpreferred_labels))) {
if (!($vs_rel_type = $va_element_data['_relationship_type']) && !($vs_rel_type = $va_element_data['idno']['_relationship_type'])) {
break;
}
$t_subject->addRelationship($vs_table_name, $vn_rel_id, trim($va_element_data['_relationship_type']), null, null, null, null, array('interstitialValues' => $va_element_data['_interstitial']));
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related movement with relationship %2:", $vs_idno, trim($va_element_data['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_item_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to mapping error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
break;
}
if (is_array($va_element_data['_related_related']) && sizeof($va_element_data['_related_related'])) {
foreach ($va_element_data['_related_related'] as $vs_rel_rel_table => $va_rel_rels) {
foreach ($va_rel_rels as $vn_i => $va_rel_rel) {
if (!($t_rel_instance = $o_dm->getInstanceByTableName($vs_table_name))) {
$o_log->logWarn(_t("[%1] Could not instantiate related table %2", $vs_idno, $vs_table_name));
continue;
}
if ($o_trans) {
$t_rel_instance->setTransaction($o_trans);
}
if ($t_rel_instance->load($vn_rel_id)) {
if ($t_rel_rel = $t_rel_instance->addRelationship($vs_rel_rel_table, $va_rel_rel['id'], $va_rel_rel['_relationship_type'])) {
$o_log->logInfo(_t('[%1] Related %2 (%3) to related %4 with relationship %5', $vs_idno, $o_dm->getTableProperty($vs_rel_rel_table, 'NAME_SINGULAR'), $va_rel_rel['id'], $t_rel_instance->getProperty('NAME_SINGULAR'), trim($va_rel_rel['_relationship_type'])));
} else {
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add related %2 (%3) to related %4 with relationship %5:", $vs_idno, $o_dm->getTableProperty($vs_rel_rel_table, 'NAME_SINGULAR'), $va_rel_rel['id'], $t_rel_instance->getProperty('NAME_SINGULAR'), trim($va_rel_rel['_relationship_type'])), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
}
}
}
}
}
}
}
}
}
// $t_subject->update();
//
// if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Invalid %2; values were %3: ", $vs_idno, 'attributes', ca_data_importers::formatValuesForLog($va_element_content)), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
// ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
// if ($vs_item_error_policy == 'stop') {
// $o_log->logAlert(_t('Import stopped due to mapping error policy'));
// if($vb_use_ncurses) { ncurses_end(); }
//
// $o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
//
// if ($o_trans) { $o_trans->rollback(); }
// return false;
// }
// }
//
$o_log->logDebug(_t('Finished inserting content tree for %1 at %2 seconds into database', $vs_idno, $t->getTime(4)));
if (!$vb_output_subject_preferred_label && $t_subject->getPreferredLabelCount() == 0) {
$t_subject->addLabel(array($vs_label_display_fld => '???'), $vn_locale_id, null, true);
if ($vs_error = DataMigrationUtils::postError($t_subject, _t("[%1] Could not add default label", $vs_idno), __CA_DATA_IMPORT_ERROR__, array('dontOutputLevel' => true, 'dontPrint' => true))) {
ca_data_importers::logImportError($vs_error, $va_log_import_error_opts);
if ($vs_import_error_policy == 'stop') {
$o_log->logAlert(_t('Import stopped due to import error policy'));
if ($vb_use_ncurses) {
ncurses_end();
}
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_FAILURE__, _t('Failed to import %1', $vs_idno));
if ($o_trans) {
$o_trans->rollback();
}
return false;
}
}
}
$o_log->logInfo(_t('[%1] Imported %2 as %3 ', $vs_idno, $vs_preferred_label_for_log, $vs_subject_table_name));
$o_event->endItem($t_subject->getPrimaryKey(), __CA_DATA_IMPORT_ITEM_SUCCESS__, _t('Imported %1', $vs_idno));
ca_data_importers::$s_num_records_processed++;
}
$o_log->logInfo(_t('Import of %1 completed using mapping %2: %3 imported/%4 skipped/%5 errors', $ps_source, $t_mapping->get('importer_code'), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_records_skipped, ca_data_importers::$s_num_import_errors));
//if ($vb_show_cli_progress_bar) {
$o_progress->finish();
//}
if ($po_request && isset($pa_options['progressCallback']) && ($ps_callback = $pa_options['progressCallback'])) {
$ps_callback($po_request, $pn_file_number, $pn_number_of_files, $ps_source, $vn_num_items, $vn_num_items, _t('Import completed'), time() - $vn_start_time, memory_get_usage(true), ca_data_importers::$s_num_records_processed, ca_data_importers::$s_num_import_errors);
}
if (isset($pa_options['reportCallback']) && ($ps_callback = $pa_options['reportCallback'])) {
$va_general = array('elapsedTime' => time() - $vn_start_time, 'numErrors' => ca_data_importers::$s_num_import_errors, 'numProcessed' => ca_data_importers::$s_num_records_processed);
$ps_callback($po_request, $va_general, ca_data_importers::$s_import_error_list, true);
}
if ($vb_use_ncurses) {
ncurses_end();
}
if ($pb_dry_run) {
if ($o_trans) {
$o_trans->rollback();
}
$o_log->logInfo(_t('Rollback successful import run in "dry run" mode'));
} else {
if ($o_trans) {
$o_trans->commit();
}
}
return true;
}
/**
* Create new entity with given user name and return id
* @param array $pa_data
* @return int|bool
*/
protected function newUser($pa_data)
{
if (!is_array($pa_data) || !isset($pa_data['username'])) {
$this->addError("Malformed request body");
return false;
}
$ps_username = $pa_data['username'];
if (ca_entities::find(array('preferred_labels' => array('surname' => $ps_username)), array('returnAs' => 'firstId'))) {
$this->addError('Username is taken');
return false;
}
$vn_new_entity_id = DataMigrationUtils::getEntityID(array('surname' => $ps_username), 'real_person', 1, array('idno' => 'public_' . time()));
if (!$vn_new_entity_id) {
return false;
}
return array('entity_id' => $vn_new_entity_id);
}
public function insert($pa_options = null)
{
if (!trim($this->get('surname')) && !trim($this->get('forename'))) {
// auto-split entity name if displayname is set
if ($vs_display_name = trim($this->get('displayname'))) {
$va_label = DataMigrationUtils::splitEntityName($vs_display_name);
if (is_array($va_label)) {
unset($va_label['displayname']);
// just make sure we don't mangle the user-entered displayname
foreach ($va_label as $vs_fld => $vs_val) {
$this->set($vs_fld, $vs_val);
}
} else {
$this->postError(1100, _t('Something went wrong when splitting displayname'), 'ca_entity_labels->insert()');
return false;
}
} else {
$this->postError(1100, _t('Surname, forename or displayname must be set'), 'ca_entity_labels->insert()');
return false;
}
}
if (!$this->get('displayname')) {
$this->set('displayname', trim(preg_replace('![ ]+!', ' ', $this->get('forename') . ' ' . $this->get('middlename') . ' ' . $this->get('surname'))));
}
return parent::insert($pa_options);
}
/**
*
*/
public function refine(&$pa_destination_data, $pa_group, $pa_item, $pa_source_data, $pa_options = null)
{
global $g_ui_locale_id;
$vs_delimiter = caGetOption('delimiter', $pa_options, null);
if (!($pn_locale_id = ca_locales::getDefaultCataloguingLocaleID())) {
$pn_locale_id = $g_ui_locale_id;
}
$o_log = isset($pa_options['log']) && is_object($pa_options['log']) ? $pa_options['log'] : null;
$t_mapping = caGetOption('mapping', $pa_options, null);
if ($t_mapping) {
$o_dm = Datamodel::load();
if ($t_mapping->get('table_num') != $o_dm->getTableNum('ca_list_items')) {
if ($o_log) {
$o_log->logError(_t("listItemIndentedHierarchyBuilder refinery may only be used in imports to ca_list_items"));
}
return null;
}
}
$va_group_dest = explode(".", $pa_group['destination']);
$vs_terminal = array_pop($va_group_dest);
$pm_value = $pa_source_data[$pa_item['source']];
// Get list of fields to insert
if (!is_array($va_levels = $pa_item['settings']['listItemIndentedHierarchyBuilder_levels'])) {
if ($o_log) {
$o_log->logError(_t("listItemIndentedHierarchyBuilder requires levels option be set to a list of data source placeholders"));
}
return null;
} else {
$va_level_types = $pa_item['settings']['listItemIndentedHierarchyBuilder_levelTypes'];
}
// Get list, or create if it doesn't already exist
if (!($vs_list_code = $pa_item['settings']['listItemIndentedHierarchyBuilder_list'])) {
if ($o_log) {
$o_log->logError(_t("listItemIndentedHierarchyBuilder requires list option be set"));
}
return null;
}
$t_list = new ca_lists();
if (!$t_list->load(array('list_code' => $vs_list_code))) {
// create list
$t_list->set('list_code', $vs_list_code);
$t_list->setMode(ACCESS_WRITE);
$t_list->insert();
if ($t_list->numErrors()) {
if ($o_log) {
$o_log->logError(_t("listItemIndentedHierarchyBuilder could not create list %1: %2", $vs_list_code, join("; ", $t_list->getErrors())));
}
return null;
}
$t_list->addLabel(array('name' => caUcFirstUTF8Safe($vs_list_code)), $pn_locale_id, null, true);
if ($t_list->numErrors()) {
if ($o_log) {
$o_log->logError(_t("listItemIndentedHierarchyBuilder could not create list label %1: %2", $vs_list_code, join("; ", $t_list->getErrors())));
}
return null;
}
}
// Handle each level
if (!is_array($va_level_values = listItemIndentedHierarchyBuilderRefinery::$opa_level_values)) {
$va_level_values = $va_level_value_ids = array();
}
$va_level_value_ids = listItemIndentedHierarchyBuilderRefinery::$opa_level_value_ids;
$vn_max_level = 0;
$vn_parent_id = null;
foreach ($va_levels as $vn_i => $vs_level_placeholder) {
$vs_level_value = null;
if (strlen($vs_level_placeholder)) {
if ($vs_level_value = BaseRefinery::parsePlaceholder($vs_level_placeholder, $pa_source_data, $pa_item, 0, array('reader' => caGetOption('reader', $pa_options, null), 'returnAsString' => true))) {
if (!$vn_parent_id && isset(listItemIndentedHierarchyBuilderRefinery::$opa_level_value_ids[$vn_i - 1])) {
$vn_parent_id = listItemIndentedHierarchyBuilderRefinery::$opa_level_value_ids[$vn_i - 1];
}
$vs_type = isset($va_level_types[$vn_i]) ? $va_level_types[$vn_i] : null;
if ($vn_item_id = DataMigrationUtils::getListItemID($vs_list_code, preg_replace("![^A-Za-z0-9_]+!", "_", $vs_level_value), $vs_type, $pn_locale_id, array('is_enabled' => 1, 'parent_id' => $vn_parent_id, 'preferred_labels' => array('name_singular' => $vs_level_value, 'name_plural' => $vs_level_value)), array('matchOnIdno' => true, 'log' => $o_log, 'transaction' => caGetOption('transaction', $pa_options, null), 'importEvent' => caGetOption('event', $pa_options, null), 'importEventSource' => 'listItemIndentedHierarchyBuilder'))) {
$vn_parent_id = $vn_item_id;
$va_level_values[$vn_i] = $vs_level_value;
$va_level_value_ids[$vn_i] = $vn_item_id;
$vn_max_level = $vn_i;
}
}
}
}
listItemIndentedHierarchyBuilderRefinery::$opa_level_values = array_slice($va_level_values, 0, $vn_max_level + 1);
listItemIndentedHierarchyBuilderRefinery::$opa_level_value_ids = array_slice($va_level_value_ids, 0, $vn_max_level + 1);
if ($pa_item['settings']['listItemIndentedHierarchyBuilder_list'] == 'returnData') {
return $vn_parent_id;
}
return null;
}
/**
*
*/
public function getIDsByLabel($pa_label_values, $pn_parent_id = null, $pn_type_id = null)
{
if (!isset($pa_label_values['forename']) && !isset($pa_label_values['surname']) && isset($pa_label_values['displayname'])) {
$pa_label_values = DataMigrationUtils::splitEntityName($pa_label_values['displayname']);
}
return $this->getEntityIDsByName($pa_label_values['forename'], $pa_label_values['surname'], $pn_parent_id, $pn_type_id);
}
/**
* Perform periodic tasks
*
* @return boolean true because otherwise it disables subsequent plugins
*/
public function hookPeriodicTask(&$pa_params)
{
global $AUTH_CURRENT_USER_ID;
$t_log = new Eventlog();
$o_db = new Db();
//$t_log->log(array('CODE' => 'ERR', 'MESSAGE' => _t('Could not authenticate to remote system %1', $vs_base_url), 'SOURCE' => 'traveloguePlugin->hookPeriodicTask'));
// Get new email
$pn_locale_id = 1;
// US
$vs_server = $this->opo_config->get('imap_server');
$vs_username = $this->opo_config->get('username');
$vs_password = $this->opo_config->get('password');
$vs_ssl = $this->opo_config->get('ssl');
if (!$vs_server) {
return true;
}
if (!$vs_username) {
return true;
}
try {
$o_mail = new Zend_Mail_Storage_Imap(array('host' => $vs_server, 'user' => $vs_username, 'password' => $vs_password, 'ssl' => $vs_ssl));
} catch (Exception $e) {
return true;
}
$va_mimetypes = $this->opo_config->getList('mimetypes');
$va_mail_to_delete = array();
foreach ($o_mail as $vn_message_num => $o_message) {
$va_mail_to_delete[$vn_message_num] = true;
// Extract title from subject line of email
$vs_subject = $o_message->subject;
$vs_from = $o_message->headerExists('from') ? $o_message->from : "";
print "PROCESSING {$vs_subject} FROM {$vs_from}\n";
// Extract media from email attachments
// Extract caption from email body
$va_images = array();
$va_texts = array();
foreach (new RecursiveIteratorIterator($o_message) as $o_part) {
try {
if (in_array(strtok($o_part->contentType, ';'), $va_mimetypes)) {
$va_images[] = $o_part;
} else {
if (in_array(strtok($o_part->contentType, ';'), array("text/plain", "text/html"))) {
$va_texts[] = (string) $o_part;
}
}
} catch (Zend_Mail_Exception $e) {
// ignore
}
}
if (!sizeof($va_images)) {
continue;
}
// Get user by email address
if (preg_match('!<([^>]+)>!', $vs_from, $va_matches)) {
// extract raw address from "from" header
$vs_from = $va_matches[1];
}
$t_user = new ca_users();
if ($t_user->load(array('email' => $vs_from))) {
$AUTH_CURRENT_USER_ID = $vn_user_id = $t_user->getPrimaryKey();
// force libs to consider matched user as logged in; change log will reflect this name
} else {
$vn_user_id = null;
}
// Create object
$t_object = new ca_objects();
$t_object->setMode(ACCESS_WRITE);
$t_object->set('type_id', $this->opo_config->get('object_type'));
// TODO: set idno to autogenerated # and/or allow for configurable policy
$t_object->set('idno', '');
$t_object->set('access', $this->opo_config->get('default_access'));
$t_object->set('status', $this->opo_config->get('default_status'));
// TODO: make this a configurable mapping ala how media metadata is done
$t_object->addAttribute(array('locale_id' => $pn_locale_id, 'generalNotes' => join("\n\n", $va_texts)), 'generalNotes');
$t_object->insert();
DataMigrationUtils::postError($t_object, "While adding object", "traveloguePlugin");
// TODO: log this
$t_object->addLabel(array('name' => $vs_subject), $pn_locale_id, null, true);
DataMigrationUtils::postError($t_object, "While adding label", "traveloguePlugin");
// TODO: log this
// Add representation
$vs_tmp_file_path = tempnam(caGetTempDirPath(), 'travelogue_');
foreach ($va_images as $vn_i => $vs_file_content) {
if (file_put_contents($vs_tmp_file_path, base64_decode((string) $vs_file_content))) {
$t_object->addRepresentation($vs_tmp_file_path, $this->opo_config->get('representation_type'), 1, $this->opo_config->get('default_status'), $this->opo_config->get('default_access'), true);
DataMigrationUtils::postError($t_object, "While adding media", "traveloguePlugin");
// TODO: log this
}
}
// TODO: add option to link user-as-entity to image (probably as creator)
}
foreach (array_reverse(array_keys($va_mail_to_delete)) as $vn_message_num) {
$o_mail->removeMessage($vn_message_num);
}
return true;
}
/**
*
*/
public function refine(&$pa_destination_data, $pa_group, $pa_item, $pa_source_data, $pa_options = null)
{
$o_log = isset($pa_options['log']) && is_object($pa_options['log']) ? $pa_options['log'] : null;
$o_trans = isset($pa_options['transaction']) && $pa_options['transaction'] instanceof Transaction ? $pa_options['transaction'] : null;
$va_group_dest = explode(".", $pa_group['destination']);
$vs_terminal = array_pop($va_group_dest);
$va_url = parse_url($pa_options['source']);
$vs_db = substr($va_url['path'], 1);
$o_db = new Db(null, array("username" => $va_url['user'], "password" => $va_url['pass'], "host" => $va_url['host'], "database" => $vs_db, "type" => 'mysql'));
parse_str($va_url['query'], $va_path);
$vs_table = $va_path['table'];
$va_entity_types = $pa_item['settings']['ATEntityGetter_entityTypes'];
//
// Grab related names from AT
//
$vs_key = $pa_item['settings']['ATEntityGetter_key'];
$qr_rel_names = $o_db->query("\n\t\t\t\tSELECT n.*, adn.* \n\t\t\t\tFROM ArchDescriptionNames adn\n\t\t\t\tINNER JOIN Names AS n ON n.nameId = adn.primaryNameId\n\t\t\t\tWHERE\n\t\t\t\t\tadn.{$vs_key} = ?\n\t\t\t", $pa_source_data[$vs_key]);
$va_vals = array();
$vn_c = 0;
//foreach($va_entities as $vn_i => $vs_entity) {
while ($qr_rel_names->nextRow()) {
// Extract entity values from record
$va_row = $qr_rel_names->getRow();
$va_split_name = array();
if ($va_row['nameType'] == 'Corporate Body') {
// corporate
$vs_entity_type = $va_entity_types['corporate'];
$va_split_name = DataMigrationUtils::splitEntityName($vs_entity = $va_row['corporatePrimaryName']);
} elseif ($va_row['nameType'] == 'Family') {
// family
$vs_entity_type = $va_entity_types['family'];
$va_split_name = DataMigrationUtils::splitEntityName($vs_entity = $va_row['familyName']);
$va_split_name['prefix'] = $va_row['familyNamePrefix'];
} else {
// personal
$vs_entity_type = $va_entity_types['personal'];
$va_split_name = array();
$va_split_name['prefix'] = $va_row['personalPrefix'];
$va_split_name['suffix'] = $va_row['personalSuffix'];
$va_split_name['forename'] = $va_row['personalRestOfName'];
$va_split_name['surname'] = $va_row['personalPrimaryName'];
}
// Import record
if (!($vs_entity = trim($vs_entity))) {
continue;
}
if (is_array($va_skip_values = $pa_item['settings']['ATEntityGetter_skipIfValue']) && in_array($vs_entity, $va_skip_values)) {
continue;
}
if (isset($va_split_name[$vs_terminal])) {
return $va_split_name[$vs_terminal];
}
if (in_array($vs_terminal, array('preferred_labels', 'nonpreferred_labels'))) {
return $va_split_name;
}
// Set label
$va_val = array('preferred_labels' => $va_split_name);
// Set entity_type
if ($vs_entity_type) {
$va_val['_type'] = BaseRefinery::parsePlaceholder($vs_entity_type, $pa_source_data, $pa_item, $vn_c);
}
if ((!isset($va_val['_type']) || !$va_val['_type']) && ($vs_type_opt = $pa_item['settings']['ATEntityGetter_entityTypeDefault'])) {
$va_val['_type'] = BaseRefinery::parsePlaceholder($vs_type_opt, $pa_source_data, $pa_item, $vs_delimiter, $vn_c);
}
// Set relationship type
if (!preg_match("!^([A-Za-z0-9 ]+)!", $va_row['role'], $va_matches) || !($va_val['_relationship_type'] = trim($va_matches[1]))) {
if ($vs_rel_type_opt = $pa_item['settings']['ATEntityGetter_relationshipTypeDefault']) {
$va_val['_relationship_type'] = BaseRefinery::parsePlaceholder($vs_rel_type_opt, $pa_source_data, $pa_item, $vs_delimiter, $vn_c);
}
}
$va_val['_relationship_type'] = str_replace(" ", "_", $va_val['_relationship_type']);
$va_val['_relationship_type'] = BaseRefinery::parsePlaceholder($va_val['_relationship_type'], $pa_source_data, $pa_item, $vs_delimiter, $vn_c);
// Set attributes
if (is_array($pa_item['settings']['ATEntityGetter_attributes'])) {
$va_attr_vals = array();
foreach ($pa_item['settings']['ATEntityGetter_attributes'] as $vs_element_code => $va_attrs) {
if (is_array($va_attrs)) {
foreach ($va_attrs as $vs_k => $vs_v) {
$va_attr_vals[$vs_element_code][$vs_k] = BaseRefinery::parsePlaceholder($vs_v, $pa_source_data, $pa_item, $vs_delimiter, $vn_c);
}
}
}
$va_val = array_merge($va_val, $va_attr_vals);
}
$va_vals[] = $va_val;
$vn_c++;
}
return $va_vals;
}
/**
* Process form submission
*/
public function Send()
{
global $g_ui_locale_id;
$o_dm = Datamodel::load();
$ps_function = $this->request->getParameter('_contributeFormName', pString);
$va_response_data = array('errors' => array(), 'numErrors' => 0, 'status' => 'OK');
$vn_num_errors = 0;
if (!($va_form_info = $this->_checkForm($ps_function))) {
return;
}
$va_related_form_item_config = caGetOption('related', $va_form_info, array());
$this->view->setVar('t_subject', $t_subject = $this->pt_subject);
$vs_idno_fld_name = $t_subject->getProperty('ID_NUMBERING_ID_FIELD');
$t_subject->clear();
$t_subject->setMode(ACCESS_WRITE);
$t_subject->purify(true);
// run all input through HTMLpurifier
$t_subject->setTransaction($o_trans = new Transaction());
$vs_subject_table = $t_subject->tableName();
$t_subject->set('type_id', $va_form_info['type'] ? $va_form_info['type'] : $this->request->getParameter('type_id', pInteger));
// set type so idno's reflect proper format
// Set window title
MetaTagManager::setWindowTitle(caGetOption('formTitle', $va_form_info, $this->request->config->get("app_display_name") . ": " . _t("Contribute")));
// Get list of form elements to process
$va_fields = explode(';', $this->request->getParameter('_formElements', pString));
// Clean up field names, which PHP has mangled by replacing periods with underscores
if (is_array($va_fields) && sizeof($va_fields) > 0) {
foreach ($va_fields as $vs_orig_fld_name) {
$vs_orig_fld_name_proc = str_replace(".", "_", $vs_orig_fld_name);
$_REQUEST[$vs_orig_fld_name] = $_REQUEST[$vs_orig_fld_name_proc];
unset($_REQUEST[$vs_orig_fld_name_proc]);
}
}
// Check terms
if (caGetOption('terms_and_conditions', $va_form_info, false)) {
// Check terms and conditions checkbox
if ($this->request->getParameter('iAgreeToTerms', pInteger) != 1) {
$this->notification->addNotification(_t("You must agree to the terms and conditions before proceeding."), __NOTIFICATION_TYPE_ERROR__);
$va_response_data['numErrors'] = 1;
$va_response_data['status'] = 'ERR';
$va_response_data['errors']['_general_'][] = _t("You must agree to the terms and conditions before proceeding.");
$va_response_data['formData'] = $_REQUEST;
$this->view->setVar('response', $va_response_data);
$t_subject->getTransaction()->rollback();
call_user_method($ps_function, $this);
return;
}
}
// Spam check
if (caGetOption('spam_protection', $va_form_info, false)) {
// Check SPAM-preventing security question
if ($this->request->getParameter('security', pInteger) != $this->request->getParameter('sum', pInteger)) {
$this->notification->addNotification(_t("Please correctly answer the security question."), __NOTIFICATION_TYPE_ERROR__);
$va_response_data['numErrors'] = 1;
$va_response_data['status'] = 'ERR';
$va_response_data['errors']['_general_'][] = _t("Please correctly answer the security question.");
$va_response_data['formData'] = $_REQUEST;
$this->view->setVar('response', $va_response_data);
$t_subject->getTransaction()->rollback();
call_user_method($ps_function, $this);
return;
}
}
// Set content from form
$vm_type = $vs_idno = $vn_status = $vn_access = null;
$vb_has_media = false;
// Assemble content tree
$va_content_tree = array();
foreach ($va_fields as $vs_field) {
$va_fld_bits = explode(".", $vs_field);
$vs_field_proc = str_replace(".", "_", $vs_field);
// PHP replaces periods in names with underscores :-(
$vs_table = $va_fld_bits[0];
$va_vals = $this->request->getParameter($vs_field_proc, pArray);
if ($vs_subject_table == $vs_table) {
// subject table
switch (sizeof($va_fld_bits)) {
case 2:
case 3:
if ($t_subject->hasField($va_fld_bits[1])) {
// intrinsic
$va_content_tree[$vs_subject_table][$va_fld_bits[1]] = $va_vals[0];
switch ($va_fld_bits[1]) {
case $t_subject->getTypeFieldName():
$vm_type = $va_vals[0];
break;
case $vs_idno_fld_name:
// parse out value
if (method_exists($t_subject, "loadIDNoPlugInInstance") && ($o_numbering_plugin = $t_subject->loadIDNoPlugInInstance(array('IDNumberingConfig' => $this->config)))) {
$vs_idno = $o_numbering_plugin->htmlFormValue($vs_idno_fld_name, null, true);
} else {
$vs_idno = $va_vals[0];
}
break;
case 'status':
$vn_status = (int) $va_vals[0];
break;
case 'access':
$vn_access = (int) $va_vals[0];
break;
}
} elseif ($va_fld_bits[1] == 'preferred_labels') {
// preferred labels
if (!isset($va_fld_bits[2])) {
$va_fld_bits[2] = $t_subject->getLabelDisplayField();
}
foreach ($va_vals as $vn_i => $vs_val) {
$va_content_tree[$vs_subject_table]['preferred_labels'][$vn_i][$va_fld_bits[2]] = $va_vals[$vn_i];
}
} elseif ($va_fld_bits[1] == 'nonpreferred_labels') {
// preferred labels
if (!isset($va_fld_bits[2])) {
$va_fld_bits[2] = $t_subject->getLabelDisplayField();
}
foreach ($va_vals as $vn_i => $vs_val) {
$va_content_tree[$vs_subject_table]['nonpreferred_labels'][$vn_i][$va_fld_bits[2]] = $va_vals[$vn_i];
}
} elseif ($t_subject->hasElement($va_fld_bits[1])) {
if (!isset($va_fld_bits[2])) {
$va_fld_bits[2] = $va_fld_bits[1];
}
if (!is_array($va_vals)) {
break;
}
foreach ($va_vals as $vn_i => $vs_val) {
$va_content_tree[$vs_subject_table][$va_fld_bits[1]][$vn_i][$va_fld_bits[2]] = $va_vals[$vn_i];
}
}
break;
}
} else {
// Process related
switch (sizeof($va_fld_bits)) {
case 2:
case 3:
if ($t_instance = $o_dm->getInstanceByTableName($vs_table, true)) {
if ($t_instance->hasField($va_fld_bits[1])) {
// intrinsic
if ($t_instance->getFieldInfo($va_fld_bits[1], 'FIELD_TYPE') == FT_MEDIA) {
$va_files = array();
foreach ($_FILES[$vs_field_proc]['tmp_name'] as $vn_index => $vs_tmp_name) {
if (!trim($vs_tmp_name)) {
continue;
}
$va_files[$vn_index] = array('tmp_name' => $vs_tmp_name, 'name' => $_FILES[$vs_field_proc]['name'][$vn_index], 'type' => $_FILES[$vs_field_proc]['type'][$vn_index], 'error' => $_FILES[$vs_field_proc]['error'][$vn_index], 'size' => $_FILES[$vs_field_proc]['size'][$vn_index]);
}
foreach ($va_files as $vn_index => $va_file) {
$va_content_tree[$vs_table][$vn_index][$va_fld_bits[1]] = $va_file;
}
} else {
foreach ($va_vals as $vn_index => $vm_val) {
$va_content_tree[$vs_table][$vn_index][$va_fld_bits[1]] = $vm_val;
}
}
} elseif ($va_fld_bits[1] == 'preferred_labels') {
// preferred labels
if (!isset($va_fld_bits[2])) {
$va_fld_bits[2] = $t_instance->getLabelDisplayField();
}
foreach ($va_vals as $vn_i => $vs_val) {
$va_content_tree[$vs_table][$vn_i]['preferred_labels'][$va_fld_bits[2]] = $va_vals[$vn_i];
}
} elseif ($va_fld_bits[1] == 'nonpreferred_labels') {
// preferred labels
if (!isset($va_fld_bits[2])) {
$va_fld_bits[2] = $t_instance->getLabelDisplayField();
}
foreach ($va_vals as $vn_i => $vs_val) {
$va_content_tree[$vs_table][$vn_i]['nonpreferred_labels'][$va_fld_bits[2]] = $va_vals[$vn_i];
}
} elseif ($t_instance->hasElement($va_fld_bits[1])) {
if (!isset($va_fld_bits[2])) {
$va_fld_bits[2] = $va_fld_bits[1];
}
foreach ($va_vals as $vn_i => $vs_val) {
$va_content_tree[$vs_table][$vn_i][$va_fld_bits[1]][$va_fld_bits[2]] = $va_vals[$vn_i];
}
}
}
if (is_array($va_rel_types = $this->request->getParameter($vs_field_proc . '_relationship_type', pArray))) {
foreach ($va_rel_types as $vn_i => $vs_rel_type) {
$va_content_tree[$vs_table][$vn_i]['_relationship_type'] = $vs_rel_type;
}
}
if (is_array($va_types = $this->request->getParameter($vs_field_proc . '_type', pArray))) {
foreach ($va_types as $vn_i => $vs_type) {
$va_content_tree[$vs_table][$vn_i]['_type'] = $vs_type;
}
}
break;
}
}
}
//print_R($va_content_tree); die;
// Set type and idno (from config or tree) and insert
// Configured values are always used in preference
foreach (array('type_id' => 'type', $vs_idno_fld_name => $vs_idno_fld_name, 'access' => 'access', 'status' => 'status') as $vs_fld => $vs_name) {
if ($vs_fld == $vs_idno_fld_name) {
$t_subject->setIdnoWithTemplate($va_form_info[$vs_idno_fld_name] ? $va_form_info[$vs_idno_fld_name] : $vs_idno, array('IDNumberingConfig' => $this->config));
} else {
$vs_varname = "vs_{$vs_name}";
$t_subject->set($vs_fld, $va_form_info[$vs_name] ? $va_form_info[$vs_name] : ${$vs_varname});
}
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
}
// Insert
$t_subject->insert();
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
// Set other content
foreach ($va_content_tree as $vs_table => $va_content_by_table) {
if ($vs_subject_table == $vs_table) {
// subject table
foreach ($va_content_by_table as $vs_bundle => $va_data_for_bundle) {
switch ($vs_bundle) {
case 'preferred_labels':
foreach ($va_data_for_bundle as $va_data) {
$t_subject->addLabel($va_data, $g_ui_locale_id, null, true);
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
}
break;
case 'nonpreferred_labels':
foreach ($va_data_for_bundle as $va_data) {
$t_subject->addLabel($va_data, $g_ui_locale_id, null, false);
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
}
break;
default:
if ($t_subject->hasField($vs_bundle) && !in_array($vs_bundle, array('type_id', $vs_idno_fld_name, 'access', 'status'))) {
$t_subject->set($vs_bundle, $va_data_for_bundle[0]);
} elseif ($t_subject->hasElement($vs_bundle)) {
foreach ($va_data_for_bundle as $va_data) {
$t_subject->addAttribute(array_merge($va_data, array('locale_id' => $g_ui_locale_id)), $vs_bundle);
}
}
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
break;
}
}
} else {
// Related table
switch ($vs_table) {
case 'ca_object_representations':
$vb_is_primary = true;
foreach ($va_content_by_table as $vn_index => $va_representation) {
if (!$va_representation['media']['tmp_name'] || $va_representation['media']['size'] == 0) {
continue;
}
if (isset($va_form_info['representation_type'])) {
$va_representation['type_id'] = $va_form_info['representation_type'];
}
if (isset($va_form_info['representation_access'])) {
$va_representation['access'] = $va_form_info['representation_access'];
}
if (isset($va_form_info['representation_status'])) {
$va_representation['status'] = $va_form_info['representation_status'];
}
$vn_rc = $t_subject->addRepresentation($va_representation['media']['tmp_name'], $va_representation['type_id'], $g_ui_locale_id, $va_representation['status'], $va_representation['access'], $vb_is_primary, $va_representation, array('original_filename' => $va_representation['media']['name']));
if ($t_subject->numErrors()) {
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
} else {
$vb_has_media = true;
$vb_is_primary = false;
}
}
break;
case 'ca_entities':
$va_rel_config = caGetOption('ca_entities', $va_related_form_item_config, array());
foreach ($va_content_by_table as $vn_index => $va_rel) {
foreach (array('idno', 'access', 'status') as $vs_f) {
$va_rel[$vs_f] = $va_rel_config[$vs_f];
}
if ($vn_rel_id = DataMigrationUtils::getEntityID(DataMigrationUtils::splitEntityName($va_rel['preferred_labels']['displayname']), $va_rel['_type'], $g_ui_locale_id, $va_rel, array('transaction' => $o_trans, 'matchOn' => array('label'), 'IDNumberingConfig' => $this->config))) {
if (!($vs_rel_type = trim($va_rel['_relationship_type']))) {
break;
}
$t_subject->addRelationship($vs_table, $vn_rel_id, $vs_rel_type);
if ($t_subject->numErrors()) {
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
}
}
}
break;
case 'ca_places':
$va_rel_config = caGetOption('ca_places', $va_related_form_item_config, array());
foreach ($va_content_by_table as $vn_index => $va_rel) {
foreach (array('idno', 'access', 'status') as $vs_f) {
$va_rel[$vs_f] = $va_rel_config[$vs_f];
}
if ($vn_rel_id = DataMigrationUtils::getPlaceID($va_rel['preferred_labels']['name'], caGetOption('parent_id', $va_rel_config, null), $va_rel['_type'], $g_ui_locale_id, $va_rel, array('transaction' => $o_trans, 'matchOn' => array('label'), 'IDNumberingConfig' => $this->config))) {
if (!($vs_rel_type = trim($va_rel['_relationship_type']))) {
break;
}
$t_subject->addRelationship($vs_table, $vn_rel_id, $vs_rel_type);
if ($t_subject->numErrors()) {
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
}
}
}
break;
case 'ca_occurrences':
$va_rel_config = caGetOption('ca_occurrences', $va_related_form_item_config, array());
foreach ($va_content_by_table as $vn_index => $va_rel) {
foreach (array('idno', 'access', 'status') as $vs_f) {
$va_rel[$vs_f] = $va_rel_config[$vs_f];
}
if ($vn_rel_id = DataMigrationUtils::getOccurrenceID($va_rel['preferred_labels']['name'], caGetOption('parent_id', $va_rel_config, null), $va_rel['_type'], $g_ui_locale_id, $va_rel, array('transaction' => $o_trans, 'matchOn' => array('label'), 'IDNumberingConfig' => $this->config))) {
if (!($vs_rel_type = trim($va_rel['_relationship_type']))) {
break;
}
$t_subject->addRelationship($vs_table, $vn_rel_id, $vs_rel_type);
if ($t_subject->numErrors()) {
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
}
}
}
break;
case 'ca_collections':
$va_rel_config = caGetOption('ca_collections', $va_related_form_item_config, array());
foreach ($va_content_by_table as $vn_index => $va_rel) {
foreach (array('idno', 'access', 'status', 'parent_id') as $vs_f) {
$va_rel[$vs_f] = $va_rel_config[$vs_f];
}
if ($vn_rel_id = DataMigrationUtils::getCollectionID($va_rel['preferred_labels']['name'], $va_rel['_type'], $g_ui_locale_id, $va_rel, array('transaction' => $o_trans, 'matchOn' => array('label'), 'IDNumberingConfig' => $this->config))) {
if (!($vs_rel_type = trim($va_rel['_relationship_type']))) {
break;
}
$t_subject->addRelationship($vs_table, $vn_rel_id, $vs_rel_type);
if ($t_subject->numErrors()) {
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
}
}
}
break;
default:
// (noop for now)
break;
}
}
}
$t_subject->update();
$this->_checkErrors($t_subject, $va_response_data, $vn_num_errors);
if ($vn_num_errors > 0) {
$va_response_data['numErrors'] = $vn_num_errors;
$va_response_data['status'] = 'ERR';
$va_response_data['formData'] = $_REQUEST;
$this->view->setVar('response', $va_response_data);
$t_subject->getTransaction()->rollback();
$this->notification->addNotification(_t('There were errors in your submission. See below for details.'), __NOTIFICATION_TYPE_ERROR__);
call_user_method($ps_function, $this);
} else {
$t_subject->getTransaction()->commit();
if (caGetOption('set_post_submission_notification', $va_form_info, false)) {
$this->notification->addNotification(caGetOption($vb_has_media ? 'post_submission_notification_message_with_media' : 'post_submission_notification_message', $va_form_info, _t('Thank you!')), __NOTIFICATION_TYPE_INFO__);
}
// Redirect to "thank you" page. Options are:
# splash = redirect to Pawtucket splash/front page
# url = redirect to Pawtucket url specified in post_submission_destination_url directive
# page = use result_html view to format direct response (no redirect)
switch ($va_form_info['post_submission_destination']) {
case 'url':
if (!is_array($va_form_info['post_submission_destination_url']) || !sizeof($va_form_info['post_submission_destination_url']) || !isset($va_form_info['post_submission_destination_url']['controller'])) {
$this->notification->addNotification(_t('No destination url configured for form %1', $ps_function), __NOTIFICATION_TYPE_ERROR__);
$this->response->setRedirect(caNavUrl($this->request, "", "Front", "Index"));
break;
}
$va_url = $va_form_info['post_submission_destination_url'];
$this->response->setRedirect(caNavUrl($this->request, $va_url['module'], $va_url['controller'], $va_url['action']));
break;
case 'page':
if (!isset($va_form_info['post_submission_view']) || !$va_form_info['post_submission_view']) {
$this->notification->addNotification(_t('No destination view configured for form %1', $ps_function), __NOTIFICATION_TYPE_ERROR__);
$this->response->setRedirect(caNavUrl($this->request, "", "Front", "Index"));
break;
}
$this->render($va_form_info['post_submission_view']);
break;
default:
case 'front':
$this->response->setRedirect(caNavUrl($this->request, "", "Front", "Index"));
break;
}
}
return;
}
/**
*
*
* @return string
*/
function caLoadULAN($ps_path_to_ulan_data = null, $ps_path_to_ulan_config = null, $pa_options = null)
{
require_once __CA_LIB_DIR__ . '/core/Db.php';
require_once __CA_LIB_DIR__ . '/core/Configuration.php';
require_once __CA_LIB_DIR__ . '/ca/Utils/DataMigrationUtils.php';
require_once __CA_MODELS_DIR__ . '/ca_locales.php';
require_once __CA_MODELS_DIR__ . '/ca_entities.php';
require_once __CA_MODELS_DIR__ . '/ca_entities_x_entities.php';
require_once __CA_MODELS_DIR__ . '/ca_lists.php';
require_once __CA_MODELS_DIR__ . '/ca_list_items.php';
require_once __CA_MODELS_DIR__ . '/ca_list_items_x_list_items.php';
require_once __CA_MODELS_DIR__ . '/ca_relationship_types.php';
$t = new Timer();
$o_log = new KLogger(__CA_APP_DIR__ . '/log', KLogger::INFO);
$va_parent_child_links = array();
$va_item_item_links = array();
$va_ulan_id_to_item_id = array();
$o_log->logInfo("Starting import of Getty ULAN");
define('__CA_DONT_DO_SEARCH_INDEXING__', true);
$_ = new Zend_Translate('gettext', __CA_APP_DIR__ . '/locale/en_US/messages.mo', 'en_US');
$t_locale = new ca_locales();
$pn_en_locale_id = $t_locale->loadLocaleByCode('en_US');
if (!($o_config = Configuration::load($ps_path_to_ulan_config))) {
$o_log->logError("Could not load ULAN import configuration file");
die("ERROR: Could not load ULAN import configuration\n");
}
$vs_ulan_import_mode = $o_config->get('ulan_import_target');
$t_list = null;
if ($vs_ulan_import_mode == 'ca_entities') {
$va_ulan_types = $o_config->getAssoc('ulan_entity_types');
$va_mapping = $o_config->getAssoc('ulan_entity_mapping');
} elseif ($vs_ulan_import_mode == 'ca_list_items') {
$va_ulan_types = $o_config->getAssoc('ulan_list_item_types');
if (!($vs_ulan_list_code = $o_config->get('ulan_import_list'))) {
$vs_ulan_list_code = 'ULAN';
}
// create vocabulary list record (if it doesn't exist already)
$t_list = new ca_lists();
if (!$t_list->load(array('list_code' => $vs_ulan_list_code))) {
$t_list->setMode(ACCESS_WRITE);
$t_list->set('list_code', $vs_ulan_list_code);
$t_list->set('is_system_list', 0);
$t_list->set('is_hierarchical', 1);
$t_list->set('use_as_vocabulary', 1);
$t_list->insert();
if ($t_list->numErrors()) {
$o_log->logError("Could not create list record for ULAN: " . join('; ', $t_list->getErrors()));
die("ERROR: couldn't create ca_list row for ULAN: " . join('; ', $t_list->getErrors()) . "\n");
}
$t_list->addLabel(array('name' => 'Union List of Artist Names'), $pn_en_locale_id, null, true);
}
$vn_list_id = $t_list->getPrimaryKey();
$va_mapping = $o_config->getAssoc('ulan_list_item_mapping');
} else {
$o_log->logError("Invalid ULAN import mode {$vs_ulan_import_mode}");
die("ERROR: invalid ULAN import mode {$vs_ulan_import_mode}\n");
}
$vn_last_message_length = 0;
$vn_term_count = 0;
$va_subject = array();
foreach (array('ULAN1.xml', 'ULAN2.xml', 'ULAN3.xml') as $vs_file) {
if (!$ps_path_to_ulan_data) {
$ps_path_to_ulan_data = ".";
}
if (!file_exists($ps_path_to_ulan_data . "/{$vs_file}")) {
$o_log->logError("Could not find ULAN data file {$vs_file}");
print "[ERROR] cannot find ULAN data.\n";
continue;
}
$o_log->logInfo("Processing ULAN file {$vs_file}");
print "[Notice] Processing ULAN file {$vs_file}\n";
// load
$o_xml = new XMLReader();
$o_xml->open($ps_path_to_ulan_data . '/' . $vs_file);
while ($o_xml->read()) {
switch ($o_xml->name) {
# ---------------------------
case 'Subject':
if ($o_xml->nodeType == XMLReader::END_ELEMENT) {
if (in_array($va_subject['subject_id'], array('500000000', '500000001'))) {
break;
}
// skip top-level root
$vs_preferred_term = $va_subject['preferred_term'];
$pb_is_enabled = false;
switch ($va_subject['record_type']) {
case 'Person':
default:
$vn_type_id = $va_ulan_types['Person'];
$pb_is_enabled = true;
break;
case 'Corporate Body':
$vn_type_id = $va_ulan_types['Corporate Body'];
$pb_is_enabled = true;
break;
}
print str_repeat(chr(8), $vn_last_message_length);
$vs_message = "\tIMPORTING #" . ($vn_term_count + 1) . " [" . $va_subject['subject_id'] . "] " . $vs_preferred_term;
if (($vn_l = 100 - strlen($vs_message)) < 1) {
$vn_l = 1;
}
$vs_message .= str_repeat(' ', $vn_l);
$vn_last_message_length = strlen($vs_message);
print $vs_message;
if ($vs_ulan_import_mode == 'ca_entities') {
$va_np_labels = array();
if (is_array($va_subject['non_preferred_terms'])) {
for ($vn_i = 0; $vn_i < sizeof($va_subject['non_preferred_terms']); $vn_i++) {
$va_np_labels[] = DataMigrationUtils::splitEntityName(trim(htmlentities($va_subject['non_preferred_terms'][$vn_i])));
}
}
$t_item = DataMigrationUtils::getEntityID(DataMigrationUtils::splitEntityName(trim(htmlentities($vs_preferred_term, ENT_NOQUOTES))), $vn_type_id, $pn_en_locale_id, array('idno' => $va_subject['subject_id']), array('nonPreferredLabels' => $va_np_labels, 'returnInstance' => true));
if (!$t_item) {
$o_log->logError("Failed to create entity for ULAN artist {$vs_preferred_term}");
break;
}
$t_item->setMode(ACCESS_WRITE);
$va_ulan_id_to_item_id[$va_subject['subject_id']] = $t_item->getPrimaryKey();
} else {
if ($t_item = $t_list->addItem($va_subject['subject_id'], $pb_is_enabled, false, null, $vn_type_id, $va_subject['subject_id'], '', 4, 1)) {
$va_ulan_id_to_item_id[$va_subject['subject_id']] = $t_item->getPrimaryKey();
if ($va_subject['preferred_parent_subject_id'] != 500000000) {
$va_parent_child_links[$va_subject['subject_id']] = $va_subject['preferred_parent_subject_id'];
}
// add preferred labels
if (!$t_item->addLabel(array('name_singular' => trim(htmlentities($vs_preferred_term, ENT_NOQUOTES)), 'name_plural' => trim(htmlentities($vs_preferred_term, ENT_NOQUOTES)), 'description' => $va_subject['description']), $pn_en_locale_id, null, true)) {
$o_log->logError("Could not add preferred label to ULAN term [" . $va_subject['subject_id'] . "] " . $vs_preferred_term . ": " . join("; ", $t_item->getErrors()));
}
// add alternate labels
if (is_array($va_subject['non_preferred_terms'])) {
for ($vn_i = 0; $vn_i < sizeof($va_subject['non_preferred_terms']); $vn_i++) {
$vs_np_label = $va_subject['non_preferred_terms'][$vn_i];
$vs_np_term_type = $va_subject['non_preferred_term_types'][$vn_i];
switch ($vs_np_term_type) {
case 'Used For Term':
$vn_np_term_type_id = $vn_list_item_label_type_uf;
break;
case 'Alternate Descriptor':
$vn_np_term_type_id = $vn_list_item_label_type_alt;
break;
default:
$vn_np_term_type_id = null;
break;
}
if (!$t_item->addLabel(array('name_singular' => trim(htmlentities($vs_np_label, ENT_NOQUOTES)), 'name_plural' => trim(htmlentities($vs_np_label, ENT_NOQUOTES)), 'description' => ''), $pn_en_locale_id, $vn_np_term_type_id, false)) {
$o_log->logError("Could not add non-preferred label to ULAN term [" . $va_subject['subject_id'] . "] " . $vs_np_label);
}
}
}
} else {
$o_log->logError("Could not import ULAN term [" . $va_subject['subject_id'] . "] " . $vs_preferred_term . ": " . join("; ", $t_list->getErrors()));
break;
}
}
// Map content fields
foreach ($va_mapping as $vs_dest => $vs_source) {
$va_values = array();
switch ($vs_source) {
case 'biography':
if (!is_array($va_subject['biographies'])) {
break;
}
foreach ($va_subject['biographies'] as $va_bio) {
$va_values[] = $va_bio['text'];
}
break;
case 'biography_dates':
if (!is_array($va_subject['biographies'])) {
break;
}
foreach ($va_subject['biographies'] as $va_bio) {
if ($va_bio['birth_date'] == 1000 || $va_bio['birth_date'] < -5000) {
if ($va_bio['death_date'] >= 2050) {
break 2;
} else {
$va_values[] = "before " . $va_bio['death_date'];
}
} elseif ($va_bio['death_date'] >= 2050) {
$va_values[] = "after " . $va_bio['birth_date'];
} else {
$va_values[] = $va_bio['birth_date'] . " - " . $va_bio['death_date'];
}
}
break;
case 'sex':
if (!is_array($va_subject['biographies'])) {
break;
}
foreach ($va_subject['biographies'] as $va_bio) {
$va_values[] = $va_bio['sex'];
}
break;
case 'nationality_name':
if (!is_array($va_subject['nationalities'])) {
break;
}
foreach ($va_subject['nationalities'] as $va_nationality) {
$va_values[] = $va_nationality['name'];
}
break;
case 'nationality_code':
if (!is_array($va_subject['nationalities'])) {
break;
}
foreach ($va_subject['nationalities'] as $va_nationality) {
$va_values[] = $va_nationality['code'];
}
break;
case 'role_name':
if (!is_array($va_subject['roles'])) {
break;
}
foreach ($va_subject['roles'] as $va_role) {
$va_values[] = $va_role['name'];
}
break;
case 'role_code':
if (!is_array($va_subject['roles'])) {
break;
}
foreach ($va_subject['roles'] as $va_role) {
$va_values[] = $va_role['code'];
}
break;
}
if (sizeof($va_values)) {
$va_dest = explode('.', $vs_dest);
$vs_fld = array_pop($va_dest);
if ($t_item->hasField($vs_fld)) {
$t_item->set($vs_fld, join("\n", $va_values));
} else {
foreach ($va_values as $vs_value) {
$t_item->addAttribute(array('locale_id' => $pn_en_locale_id, $vs_fld => $vs_value), $vs_fld);
}
}
$t_item->update(array('dontCheckCircularReferences' => true, 'dontSetHierarchicalIndexing' => true));
if ($t_item->numErrors()) {
$o_log->logError("Could not update ULAN list item with content values: " . join("; ", $t_item->getErrors()));
}
}
}
// record item-item relations
if (is_array($va_subject['related_subjects'])) {
foreach ($va_subject['related_subjects'] as $vs_rel_subject_id) {
$va_item_item_links[$va_subject['subject_id']] = $vs_rel_subject_id;
}
}
$vn_term_count++;
} else {
$va_subject = array('subject_id' => $o_xml->getAttribute('Subject_ID'));
}
break;
# ---------------------------
# ---------------------------
case 'Biographies':
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Preferred_Biography':
$va_bio = array();
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Biography_Text':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_bio['text'] = $o_xml->value;
break;
}
break;
case 'Birth_Date':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_bio['birth_date'] = $o_xml->value;
if ($va_bio['birth_date'] < 0) {
$va_bio['birth_date'] = abs($va_bio['birth_date']) . " BCE";
}
break;
}
break;
case 'Death_Date':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_bio['death_date'] = $o_xml->value;
if ($va_bio['death_date'] < 0) {
$va_bio['death_date'] = abs($va_bio['death_date']) . " BCE";
}
break;
}
break;
case 'Sex':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_bio['sex'] = $o_xml->value;
break;
}
break;
case 'Preferred_Biography':
break 2;
}
}
$va_subject['biographies'][] = $va_bio;
break;
case 'Biographies':
break 2;
}
}
break;
# ---------------------------
# ---------------------------
case 'Nationalities':
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Preferred_Nationality':
$va_nationality = array();
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Nationality_Code':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_nationality['code'] = $o_xml->value;
$va_nationality['name'] = array_pop(explode('/', $o_xml->value));
break;
}
break;
case 'Preferred_Nationality':
break 2;
}
}
$va_subject['nationalities'][] = $va_nationality;
break;
case 'Nationalities':
break 2;
}
}
break;
# ---------------------------
# ---------------------------
case 'Roles':
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Preferred_Role':
$va_role = array();
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Role_ID':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_role['code'] = $o_xml->value;
$va_role['name'] = array_pop(explode('/', $o_xml->value));
break;
}
break;
case 'Preferred_Role':
break 2;
}
}
$va_subject['roles'][] = $va_role;
break;
case 'Roles':
break 2;
}
}
break;
# ---------------------------
# ---------------------------
case 'Record_Type':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['record_type'] = $o_xml->value;
break;
}
break;
# ---------------------------
# ---------------------------
case 'Hierarchy':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['hierarchy'] = $o_xml->value;
break;
}
break;
# ---------------------------
# ---------------------------
case 'Parent_Relationships':
$vn_parent_id = $vs_historic_flag = null;
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Preferred_Parent':
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Parent_Subject_ID':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$vn_parent_id = $o_xml->value;
break;
}
break;
case 'Historic_Flag':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$vs_historic_flag = $o_xml->value;
break;
}
break;
case 'Preferred_Parent':
$va_subject['preferred_parent_subject_id'] = $vn_parent_id;
break 2;
}
}
break;
case 'Parent_Relationships':
break 2;
}
}
break;
# ---------------------------
# ---------------------------
case 'Preferred_Term':
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Term_Type':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['preferred_term_type'] = $o_xml->value;
break;
}
break;
case 'Term_Text':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['preferred_term'] = $o_xml->value;
break;
}
break;
case 'Term_ID':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['preferred_term_id'] = $o_xml->value;
break;
}
break;
break;
case 'Preferred_Term':
break 2;
}
}
break;
# ---------------------------
# ---------------------------
case 'Non-Preferred_Term':
while ($o_xml->read()) {
switch ($o_xml->name) {
case 'Term_Type':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['non_preferred_term_types'][] = $o_xml->value;
break;
}
break;
case 'Term_Text':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['non_preferred_terms'][] = $o_xml->value;
break;
}
break;
case 'Term_ID':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['non_preferred_term_ids'][] = $o_xml->value;
break;
}
break;
case 'Non-Preferred_Term':
break 2;
}
}
break;
# ---------------------------
# ---------------------------
case 'VP_Subject_ID':
switch ($o_xml->nodeType) {
case XMLReader::ELEMENT:
$o_xml->read();
$va_subject['related_subjects'][] = $o_xml->value;
break;
}
break;
# ---------------------------
}
}
$o_xml->close();
}
$o_log->logInfo("Begin linking ULAN terms in hierarchy");
print "\n\nLINKING TERMS IN HIERARCHY...\n";
$vn_last_message_length = 0;
$t_list = new ca_lists();
$t_item = new ca_list_items();
$t_item->setMode(ACCESS_WRITE);
$vn_list_root_id = $t_list->getRootListItemID($vn_list_id);
foreach ($va_parent_child_links as $vs_child_id => $vs_parent_id) {
print str_repeat(chr(8), $vn_last_message_length);
$vs_message = "\tLINKING {$vs_child_id} to parent {$vs_parent_id}";
if (($vn_l = 100 - strlen($vs_message)) < 1) {
$vn_l = 1;
}
$vs_message .= str_repeat(' ', $vn_l);
$vn_last_message_length = strlen($vs_message);
print $vs_message;
if (in_array($vs_parent_id, array('500000000', '500000001'))) {
if (!$t_item->load($vn_child_item_id)) {
$o_log->logError("Could not load item for {$vs_child_id} (was translated to item_id={$vn_child_item_id})");
continue;
}
$t_item->set('parent_id', $vn_list_root_id);
$t_item->update(array('dontCheckCircularReferences' => true, 'dontSetHierarchicalIndexing' => true));
if ($t_item->numErrors()) {
$o_log->logError("Could not set parent_id for {$vs_child_id} to root): " . join('; ', $t_item->getErrors()));
continue;
}
$va_ulan_id_to_item_id[$vs_parent_id] = $vn_list_root_id;
}
if (!($vn_child_item_id = $va_ulan_id_to_item_id[$vs_child_id])) {
$o_log->logError("No list item id for child_id {$vs_child_id} (were there previous errors?)");
continue;
}
if (!($vn_parent_item_id = $va_ulan_id_to_item_id[$vs_parent_id])) {
$o_log->logError("No list item id for parent_id {$vs_parent_id} (were there previous errors?)");
continue;
}
if (!$t_item->load($vn_child_item_id)) {
$o_log->logError("Could not load item for {$vs_child_id} (was translated to item_id={$vn_child_item_id})");
continue;
}
$t_item->set('parent_id', $vn_parent_item_id);
$t_item->update(array('dontCheckCircularReferences' => true, 'dontSetHierarchicalIndexing' => true));
if ($t_item->numErrors()) {
$o_log->logError("Could not set parent_id for {$vs_child_id} (was translated to item_id={$vn_child_item_id}): " . join('; ', $t_item->getErrors()));
}
}
if ($vn_list_item_relation_type_id_related > 0) {
$o_log->logInfo("Begin adding ULAN related term links");
$vn_last_message_length = 0;
$t_item = new ca_list_items();
$t_link = new ca_list_items_x_list_items();
$t_link->setMode(ACCESS_WRITE);
foreach ($va_item_item_links as $vs_left_id => $vs_right_id) {
print str_repeat(chr(8), $vn_last_message_length);
$vs_message = "\tLINKING {$vs_left_id} to {$vs_right_id}";
if (($vn_l = 100 - strlen($vs_message)) < 1) {
$vn_l = 1;
}
$vs_message .= str_repeat(' ', $vn_l);
$vn_last_message_length = strlen($vs_message);
print $vs_message;
if (!($vn_left_item_id = $va_ulan_id_to_item_id[$vs_left_id])) {
$o_log->logError("No list item id for left_id {$vs_left_id} (were there previous errors?)");
continue;
}
if (!($vn_right_item_id = $va_ulan_id_to_item_id[$vs_right_id])) {
$o_log->logError("No list item id for right_id {$vs_right_id} (were there previous errors?)");
continue;
}
$t_link->set('term_left_id', $vn_left_item_id);
$t_link->set('term_right_id', $vn_right_item_id);
$t_link->set('type_id', $vn_list_item_relation_type_id_related);
$t_link->insert();
if ($t_link->numErrors()) {
$o_log->logError("Could not set link between {$vs_left_id} (was translated to item_id={$vn_left_item_id}) and {$vs_right_id} (was translated to item_id={$vn_right_item_id}): " . join('; ', $t_link->getErrors()));
}
}
} else {
$o_log->logWarn("Skipped import of term-term relationships because the ca_list_items_x_list_items 'related' relationship type is not defined for your installation");
}
$vn_duration = $t->getTime(1);
$vs_time = caFormatInterval($vn_duration);
$o_log->logInfo("Rebuilding hierarchical indices...");
$t_item->rebuildAllHierarchicalIndexes();
$o_log->logInfo("ULAN import complete. Took {$vs_time} ({$vn_duration})");
print "\n\nIMPORT COMPLETE. Took {$vs_time} ({$vn_duration})\n";
}
/**
*
*/
public function refine(&$pa_destination_data, $pa_group, $pa_item, $pa_source_data, $pa_options = null)
{
$o_log = isset($pa_options['log']) && is_object($pa_options['log']) ? $pa_options['log'] : null;
$va_group_dest = explode(".", $pa_group['destination']);
$vs_terminal = array_pop($va_group_dest);
$pm_value = $pa_source_data[$pa_item['source']];
if (is_array($pm_value)) {
$va_entities = $pm_value;
// for input formats that support repeating values
} else {
$va_entities = array($pm_value);
}
$va_vals = array();
$vn_c = 0;
$t_entity = new ca_entities();
foreach ($va_entities as $pm_value) {
if (!($vs_entity = trim($pm_value))) {
return array();
}
if (is_array($va_skip_values = $pa_item['settings']['entityJoiner_skipIfValue']) && in_array($vs_entity, $va_skip_values)) {
return array();
}
$va_name = array();
foreach ($t_entity->getLabelUIFields() as $vs_fld) {
$va_name[$vs_fld] = BaseRefinery::parsePlaceholder($pa_item['settings']['entityJoiner_' . $vs_fld], $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'returnAsString' => true, 'delimiter' => ' '));
}
if (isset($va_name[$vs_terminal])) {
return $va_name[$vs_terminal];
}
if (in_array($vs_terminal, array('preferred_labels', 'nonpreferred_labels'))) {
return $va_name;
}
// Set label
$va_val = array('preferred_labels' => $va_name);
// Set relationship type
if ($vs_rel_type_opt = $pa_item['settings']['entityJoiner_relationshipType']) {
$va_val['_relationship_type'] = BaseRefinery::parsePlaceholder($vs_rel_type_opt, $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'returnAsString' => true, 'delimiter' => ' '));
}
if ((!isset($va_val['_relationship_type']) || !$va_val['_relationship_type']) && ($vs_rel_type_opt = $pa_item['settings']['entityJoiner_relationshipTypeDefault'])) {
$va_val['_relationship_type'] = BaseRefinery::parsePlaceholder($vs_rel_type_opt, $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'returnAsString' => true, 'delimiter' => ' '));
}
if ((!isset($va_val['_relationship_type']) || !$va_val['_relationship_type']) && $o_log) {
$o_log->logWarn(_t('[entityJoinerRefinery] No relationship type is set for entity %1', $vs_entity));
}
// Set entity_type
if ($vs_type_opt = $pa_item['settings']['entityJoiner_entityType']) {
$va_val['_type'] = BaseRefinery::parsePlaceholder($vs_type_opt, $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'returnAsString' => true, 'delimiter' => ' '));
}
if ((!isset($va_val['_type']) || !$va_val['_type']) && ($vs_type_opt = $pa_item['settings']['entityJoiner_entityTypeDefault'])) {
$va_val['_type'] = BaseRefinery::parsePlaceholder($vs_type_opt, $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'returnAsString' => true, 'delimiter' => ' '));
}
if ((!isset($va_val['_type']) || !$va_val['_type']) && $o_log) {
$o_log->logWarn(_t('[entityJoinerRefinery] No entity type is set for entity %1', $vs_entity));
}
// Set attributes
if (is_array($va_attr_vals = caProcessRefineryAttributes($pa_item['settings']['entityJoiner_attributes'], $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'log' => $o_log)))) {
$va_val = array_merge($va_val, $va_attr_vals);
}
// Set interstitials
if (isset($pa_options['mapping']) && is_array($va_attr_vals = caProcessInterstitialAttributes('entityJoiner', $pa_options['mapping']->get('table_num'), 'ca_entities', $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'log' => $o_log)))) {
$va_val = array_merge($va_val, $va_attr_vals);
}
caProcessRefineryRelatedMultiple($this, $pa_item, $pa_source_data, $vn_c, $o_log, caGetOption('reader', $pa_options, null), $va_val, $va_vals);
// nonpreferred labels
if (is_array($pa_item['settings']['entityJoiner_nonpreferred_labels'])) {
$va_non_preferred_labels = array();
foreach ($pa_item['settings']['entityJoiner_nonpreferred_labels'] as $vn_index => $va_elements) {
if (is_array($va_elements)) {
$vb_non_pref_label_was_set = false;
foreach ($va_elements as $vs_k => $vs_v) {
if (!trim($vs_v)) {
continue;
}
if ($vs_k == 'split') {
if (!is_array($va_non_preferred_labels[$vn_index])) {
$va_non_preferred_labels[$vn_index] = array();
}
if ($vs_name = BaseRefinery::parsePlaceholder($vs_v, $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'returnAsString' => true, 'delimiter' => ' '))) {
$va_non_preferred_labels[$vn_index] = array_merge($va_non_preferred_labels[$vn_index], DataMigrationUtils::splitEntityName($vs_name));
$vb_non_pref_label_was_set = true;
}
} else {
if ($va_non_preferred_labels[$vn_index][$vs_k] = trim(BaseRefinery::parsePlaceholder($vs_v, $pa_source_data, $pa_item, $vn_c, array('reader' => caGetOption('reader', $pa_options, null), 'returnAsString' => true, 'delimiter' => ' ')))) {
$vb_non_pref_label_was_set = true;
}
}
}
}
if (!$vb_non_pref_label_was_set) {
unset($va_non_preferred_labels[$vn_index]);
}
}
if (sizeof($va_non_preferred_labels)) {
$va_val['nonpreferred_labels'] = $va_non_preferred_labels;
}
}
$va_vals[] = $va_val;
$vn_c++;
}
return $va_vals;
}