/**
*
*/
public static function import_data($po_opts = null)
{
require_once __CA_MODELS_DIR__ . "/ca_data_importers.php";
if (!($vs_data_source = $po_opts->getOption('source'))) {
CLIUtils::addError(_t('You must specify a data source for import'));
return false;
}
if (!$vs_data_source) {
CLIUtils::addError(_t('You must specify a source'));
return false;
}
if (!($vs_mapping = $po_opts->getOption('mapping'))) {
CLIUtils::addError(_t('You must specify a mapping'));
return false;
}
if (!ca_data_importers::mappingExists($vs_mapping)) {
CLIUtils::addError(_t('Mapping %1 does not exist', $vs_mapping));
return false;
}
$vb_no_ncurses = (bool) $po_opts->getOption('disable-ncurses');
$vs_format = $po_opts->getOption('format');
$vs_log_dir = $po_opts->getOption('log');
$vn_log_level = CLIUtils::import_getLogLevel($po_opts);
if (!ca_data_importers::importDataFromSource($vs_data_source, $vs_mapping, array('format' => $vs_format, 'showCLIProgressBar' => true, 'useNcurses' => !$vb_no_ncurses && caCLIUseNcurses(), 'logDirectory' => $vs_log_dir, 'logLevel' => $vn_log_level))) {
CLIUtils::addError(_t("Could not import source %1", $vs_data_source));
return false;
} else {
CLIUtils::addMessage(_t("Imported data from source %1", $vs_data_source));
return true;
}
}
/**
* @param Zend_Console_Getopt|null $po_opts
* @return bool
*/
public static function reload_ulan_records($po_opts = null)
{
require_once __CA_MODELS_DIR__ . '/ca_data_importers.php';
if (!($vs_mapping = $po_opts->getOption('mapping'))) {
CLIUtils::addError("\t\tNo mapping found. Please use the -m parameter to specify a ULAN mapping.");
return false;
}
if (!ca_data_importers::mappingExists($vs_mapping)) {
CLIUtils::addError("\t\tMapping {$vs_mapping} does not exist");
return false;
}
$vs_log_dir = $po_opts->getOption('log');
$vn_log_level = CLIUtils::getLogLevel($po_opts);
$o_db = new Db();
$qr_items = $o_db->query("\n\t\t\t\tSELECT DISTINCT source FROM ca_data_import_events WHERE type_code = 'ULAN'\n\t\t\t");
$va_sources = array();
while ($qr_items->nextRow()) {
$vs_source = $qr_items->get('source');
if (!isURL($vs_source)) {
continue;
}
if (!preg_match("/http\\:\\/\\/vocab\\.getty\\.edu\\/ulan\\//", $vs_source)) {
continue;
}
$va_sources[] = $vs_source;
}
ca_data_importers::importDataFromSource(join(',', $va_sources), $vs_mapping, array('format' => 'ULAN', 'showCLIProgressBar' => true, 'logDirectory' => $vs_log_dir, 'logLevel' => $vn_log_level));
return true;
}
/**
*
*/
public static function import_data($po_opts = null)
{
require_once __CA_MODELS_DIR__ . "/ca_data_importers.php";
if (!($vs_data_source = $po_opts->getOption('source'))) {
CLIUtils::addError(_t('You must specify a data source for import'));
return false;
}
if (!$vs_data_source) {
CLIUtils::addError(_t('You must specify a source'));
return false;
}
if (!($vs_mapping = $po_opts->getOption('mapping'))) {
CLIUtils::addError(_t('You must specify a mapping'));
return false;
}
if (!ca_data_importers::mappingExists($vs_mapping)) {
CLIUtils::addError(_t('Mapping %1 does not exist', $vs_mapping));
return false;
}
$vb_no_ncurses = (bool) $po_opts->getOption('disable-ncurses');
$vs_format = $po_opts->getOption('format');
$vs_log_dir = $po_opts->getOption('log');
$vn_log_level = KLogger::INFO;
switch ($vs_log_level = $po_opts->getOption('log-level')) {
case 'DEBUG':
$vn_log_level = KLogger::DEBUG;
break;
case 'NOTICE':
$vn_log_level = KLogger::NOTICE;
break;
case 'WARN':
$vn_log_level = KLogger::WARN;
break;
case 'ERR':
$vn_log_level = KLogger::ERR;
break;
case 'CRIT':
$vn_log_level = KLogger::CRIT;
break;
case 'ALERT':
$vn_log_level = KLogger::ALERT;
break;
default:
case 'INFO':
$vn_log_level = KLogger::INFO;
break;
}
if (!ca_data_importers::importDataFromSource($vs_data_source, $vs_mapping, array('format' => $vs_format, 'showCLIProgressBar' => true, 'useNcurses' => !$vb_no_ncurses && caCLIUseNcurses(), 'logDirectory' => $vs_log_dir, 'logLevel' => $vn_log_level))) {
CLIUtils::addError(_t("Could not import source %1", $vs_data_source));
return false;
} else {
CLIUtils::addMessage(_t("Imported data from source %1", $vs_data_source));
return true;
}
}
/**
* Import metadata using a mapping
*
* @param RequestHTTP $po_request The current request
* @param string $ps_source A path to a file or directory of files to import
* @param string $ps_importer The code of the importer (mapping) to use
* @param string $ps_input_format The format of the source data
* @param array $pa_options
* progressCallback =
* reportCallback =
* sendMail =
* dryRun =
* importAllDatasets =
* log = log directory path
* logLevel = KLogger constant for minimum log level to record. Default is KLogger::INFO. Constants are, in descending order of shrillness:
* KLogger::EMERG = Emergency messages (system is unusable)
* KLogger::ALERT = Alert messages (action must be taken immediately)
* KLogger::CRIT = Critical conditions
* KLogger::ERR = Error conditions
* KLogger::WARN = Warnings
* KLogger::NOTICE = Notices (normal but significant conditions)
* KLogger::INFO = Informational messages
* KLogger::DEBUG = Debugging messages
*/
public static function importMetadata($po_request, $ps_source, $ps_importer, $ps_input_format, $pa_options = null)
{
$va_errors = $va_noticed = array();
$vn_start_time = time();
$o_config = Configuration::load();
if (!ca_data_importers::mappingExists($ps_importer)) {
$va_errors['general'] = array('idno' => "*", 'label' => "*", 'errors' => array(_t('Importer %1 does not exist', $ps_importer)), 'status' => 'ERROR');
return false;
}
$vs_log_dir = caGetOption('log', $pa_options, null);
$vs_log_level = caGetOption('logLevel', $pa_options, "INFO");
$vb_import_all_datasets = caGetOption('importAllDatasets', $pa_options, false);
$vb_dry_run = caGetOption('dryRun', $pa_options, false);
$vn_log_level = BatchProcessor::_logLevelStringToNumber($vs_log_level);
if (!isURL($ps_source) && is_dir($ps_source)) {
$va_sources = caGetDirectoryContentsAsList($ps_source, true, false, false, false);
} else {
$va_sources = array($ps_source);
}
$vn_file_num = 0;
foreach ($va_sources as $vs_source) {
$vn_file_num++;
if (!ca_data_importers::importDataFromSource($vs_source, $ps_importer, array('fileNumber' => $vn_file_num, 'numberOfFiles' => sizeof($va_sources), 'logDirectory' => $o_config->get('batch_metadata_import_log_directory'), 'request' => $po_request, 'format' => $ps_input_format, 'showCLIProgressBar' => false, 'useNcurses' => false, 'progressCallback' => isset($pa_options['progressCallback']) ? $pa_options['progressCallback'] : null, 'reportCallback' => isset($pa_options['reportCallback']) ? $pa_options['reportCallback'] : null, 'logDirectory' => $vs_log_dir, 'logLevel' => $vn_log_level, 'dryRun' => $vb_dry_run, 'importAllDatasets' => $vb_import_all_datasets))) {
$va_errors['general'][] = array('idno' => "*", 'label' => "*", 'errors' => array(_t("Could not import source %1", $ps_source)), 'status' => 'ERROR');
return false;
} else {
$va_notices['general'][] = array('idno' => "*", 'label' => "*", 'errors' => array(_t("Imported data from source %1", $ps_source)), 'status' => 'SUCCESS');
//return true;
}
}
$vn_elapsed_time = time() - $vn_start_time;
if (isset($pa_options['sendMail']) && $pa_options['sendMail']) {
if ($vs_email = trim($po_request->user->get('email'))) {
caSendMessageUsingView($po_request, array($vs_email => $po_request->user->get('fname') . ' ' . $po_request->user->get('lname')), __CA_ADMIN_EMAIL__, _t('[%1] Batch metadata import completed', $po_request->config->get('app_display_name')), 'batch_metadata_import_completed.tpl', array('notices' => $va_notices, 'errors' => $va_errors, 'numErrors' => sizeof($va_errors), 'numProcessed' => sizeof($va_notices), 'subjectNameSingular' => _t('row'), 'subjectNamePlural' => _t('rows'), 'startedOn' => caGetLocalizedDate($vn_start_time), 'completedOn' => caGetLocalizedDate(time()), 'elapsedTime' => caFormatInterval($vn_elapsed_time)));
}
}
if (isset($pa_options['sendSMS']) && $pa_options['sendSMS']) {
SMS::send($po_request->getUserID(), _t("[%1] Metadata import processing for begun at %2 is complete", $po_request->config->get('app_display_name'), caGetLocalizedDate($vn_start_time)));
}
return array('errors' => $va_errors, 'notices' => $va_notices, 'processing_time' => caFormatInterval($vn_elapsed_time));
}
/**
* @param array $pa_options
* progressCallback =
* reportCallback =
* sendMail =
* log = log directory path
* logLevel = KLogger loglevel. Default is "INFO"
*/
public static function importMetadata($po_request, $ps_source, $ps_importer, $ps_input_format, $pa_options = null)
{
$va_errors = $va_noticed = array();
$vn_start_time = time();
$o_config = Configuration::load();
if (!ca_data_importers::mappingExists($ps_importer)) {
$va_errors['general'] = array('idno' => "*", 'label' => "*", 'errors' => array(_t('Importer %1 does not exist', $ps_importer)), 'status' => 'ERROR');
return false;
}
$vs_log_dir = isset($pa_options['log']) ? $pa_options['log'] : null;
$vn_log_level = KLogger::INFO;
switch ($vs_log_level = isset($pa_options['logLevel']) ? $pa_options['logLevel'] : "INFO") {
case 'DEBUG':
$vn_log_level = KLogger::DEBUG;
break;
case 'NOTICE':
$vn_log_level = KLogger::NOTICE;
break;
case 'WARN':
$vn_log_level = KLogger::WARN;
break;
case 'ERR':
$vn_log_level = KLogger::ERR;
break;
case 'CRIT':
$vn_log_level = KLogger::CRIT;
break;
case 'ALERT':
$vn_log_level = KLogger::ALERT;
break;
default:
case 'INFO':
$vn_log_level = KLogger::INFO;
break;
}
if (!ca_data_importers::importDataFromSource($ps_source, $ps_importer, array('logDirectory' => $o_config->get('batch_metadata_import_log_directory'), 'request' => $po_request, 'format' => $ps_input_format, 'showCLIProgressBar' => false, 'useNcurses' => false, 'progressCallback' => isset($pa_options['progressCallback']) ? $pa_options['progressCallback'] : null, 'reportCallback' => isset($pa_options['reportCallback']) ? $pa_options['reportCallback'] : null, 'logDirectory' => $vs_log_dir, 'logLevel' => $vn_log_level))) {
$va_errors['general'] = array('idno' => "*", 'label' => "*", 'errors' => array(_t("Could not import source %1", $vs_data_source)), 'status' => 'ERROR');
return false;
} else {
$va_notices['general'] = array('idno' => "*", 'label' => "*", 'errors' => array(_t("Imported data from source %1", $vs_data_source)), 'status' => 'SUCCESS');
//return true;
}
$vn_elapsed_time = time() - $vn_start_time;
if (isset($pa_options['sendMail']) && $pa_options['sendMail']) {
if ($vs_email = trim($po_request->user->get('email'))) {
caSendMessageUsingView($po_request, array($vs_email => $po_request->user->get('fname') . ' ' . $po_request->user->get('lname')), __CA_ADMIN_EMAIL__, _t('[%1] Batch metadata import completed', $po_request->config->get('app_display_name')), 'batch_metadata_import_completed.tpl', array('notices' => $va_notices, 'errors' => $va_errors, 'numErrors' => sizeof($va_errors), 'numProcessed' => sizeof($va_notices), 'subjectNameSingular' => _t('row'), 'subjectNamePlural' => _t('rows'), 'startedOn' => $vs_started_on, 'completedOn' => caGetLocalizedDate(time()), 'elapsedTime' => caFormatInterval($vn_elapsed_time)));
}
}
if (isset($pa_options['sendSMS']) && $pa_options['sendSMS']) {
SMS::send($po_request->getUserID(), _t("[%1] Metadata import processing for begun at %2 is complete", $po_request->config->get('app_display_name'), $vs_started_on));
}
return array('errors' => $va_errors, 'notices' => $va_notices, 'processing_time' => caFormatInterval($vn_elapsed_time));
}
/**
* Ajax-invoked execution of import process. This is where the import is actually run.
*/
public function RunImport()
{
if (!$this->request->user->canDoAction('can_import_worldcat')) {
return;
}
$pa_worldcat_ids = $this->request->getParameter('WorldCatID', pArray);
$pn_importer_id = $this->request->getParameter('importer_id', pInteger);
$ps_job_id = $this->request->getParameter('job_id', pString);
$pn_log_level = $this->request->getParameter('log_level', pInteger);
$o_progress = new ProgressBar('WebUI', 0, $ps_job_id);
$o_progress->setJobID($ps_job_id);
$o_progress->setMode('WebUI');
$o_progress->setTotal(sizeof($pa_worldcat_ids));
$vn_status = ca_data_importers::importDataFromSource(join(",", $pa_worldcat_ids), $pn_importer_id, array('progressBar' => $o_progress, 'format' => 'WorldCat', 'logLevel' => $pn_log_level));
$this->view->setVar('info', array('status' => $vn_status, 'job_id' => $ps_job_id, 'importer_id' => $pn_importer_id, 'worldcat_ids' => $pa_worldcat_ids));
$this->render('import_run_json.php');
}
