/** * Warn the user if VUFIND_LOCAL_DIR is not set. * * @return void */ protected function checkLocalSetting() { if (!getenv('VUFIND_LOCAL_DIR')) { Console::writeLine("WARNING: The VUFIND_LOCAL_DIR environment variable is not set."); Console::writeLine("This should point to your local configuration directory (i.e."); Console::writeLine(realpath(APPLICATION_PATH . '/local') . ")."); Console::writeLine("Without it, inappropriate default settings may be loaded."); Console::writeLine(""); } }
/** * XSLT Import Tool * * @return void */ public function importXslAction() { // Parse switches: $this->consoleOpts->addRules(array('test-only' => 'Use test mode', 'index-s' => 'Solr index to use')); $testMode = $this->consoleOpts->getOption('test-only') ? true : false; $index = $this->consoleOpts->getOption('index'); if (empty($index)) { $index = 'Solr'; } // Display help message if parameters missing: $argv = $this->consoleOpts->getRemainingArgs(); if (!isset($argv[1])) { Console::writeLine("Usage: import-xsl.php [--test-only] [--index <type>] " . "XML_file properties_file"); Console::writeLine("\tXML_file - source file to index"); Console::writeLine("\tproperties_file - import configuration file"); Console::writeLine("If the optional --test-only flag is set, " . "transformed XML will be displayed"); Console::writeLine("on screen for debugging purposes, " . "but it will not be indexed into VuFind."); Console::writeLine(""); Console::writeLine("If the optional --index parameter is set, " . "it must be followed by the name of"); Console::writeLine("a class for accessing Solr; it defaults to the " . "standard Solr class, but could"); Console::writeLine("be overridden with, for example, SolrAuth to " . "load authority records."); Console::writeLine(""); Console::writeLine("Note: See vudl.properties and ojs.properties " . "for configuration examples."); return $this->getFailureResponse(); } // Try to import the document if successful: try { $importer = new Importer(); $importer->setServiceLocator($this->getServiceLocator()); $importer->save($argv[0], $argv[1], $index, $testMode); } catch (\Exception $e) { Console::writeLine("Fatal error: " . $e->getMessage()); return $this->getFailureResponse(); } if (!$testMode) { Console::writeLine("Successfully imported {$argv[0]}..."); } return $this->getSuccessResponse(); }
/** * Write a string w/newline to the Console. * * @param string $str String to write. * * @return void */ protected function writeLine($str) { // Bypass output when testing: if (defined('VUFIND_PHPUNIT_RUNNING')) { return; } Console::writeLine($str); }
/** * Support method for webcrawlAction(). * * Process a sitemap URL, either harvesting its contents directly or recursively * reading in child sitemaps. * * @param string $url URL of sitemap to read. * @param bool $verbose Are we in verbose mode? * @param string $index Solr index to update * @param bool $testMode Are we in test mode? * * @return bool True on success, false on error. */ protected function harvestSitemap($url, $verbose = false, $index = 'SolrWeb', $testMode = false) { if ($verbose) { Console::writeLine("Harvesting {$url}..."); } $retVal = true; $file = tempnam('/tmp', 'sitemap'); file_put_contents($file, file_get_contents($url)); $xml = simplexml_load_file($file); if ($xml) { // Are there any child sitemaps? If so, pull them in: $results = isset($xml->sitemap) ? $xml->sitemap : []; foreach ($results as $current) { if (isset($current->loc)) { $success = $this->harvestSitemap((string) $current->loc, $verbose, $index, $testMode); if (!$success) { $retVal = false; } } } // Only import the current sitemap if it contains URLs! if (isset($xml->url)) { try { $this->performImport($file, 'sitemap.properties', $index, $testMode); } catch (\Exception $e) { if ($verbose) { Console::writeLine(get_class($e) . ': ' . $e->getMessage()); } $retVal = false; } } } unlink($file); return $retVal; }
/** * Abstract delete method. * * @param string $table Table to operate on. * @param string $successString String for reporting success. * @param string $failString String for reporting failure. * @param int $minAge Minimum age allowed for expiration (also used * as default value). * * @return mixed */ protected function expire($table, $successString, $failString, $minAge = 2) { // Get command-line arguments $argv = $this->consoleOpts->getRemainingArgs(); // Use command line value as expiration age, or default to $minAge. $daysOld = isset($argv[0]) ? intval($argv[0]) : $minAge; // Abort if we have an invalid expiration age. if ($daysOld < 2) { Console::writeLine(str_replace('%%age%%', $minAge, 'Expiration age must be at least %%age%% days.')); return $this->getFailureResponse(); } // Delete the expired searches--this cleans up any junk left in the database // from old search histories that were not // caught by the session garbage collector. $search = $this->getTable($table); if (!method_exists($search, 'getExpiredQuery')) { throw new \Exception($table . ' does not support getExpiredQuery()'); } $query = $search->getExpiredQuery($daysOld); if (($count = count($search->select($query))) == 0) { Console::writeLine($failString); return $this->getSuccessResponse(); } $search->delete($query); Console::writeLine(str_replace('%%count%%', $count, $successString)); return $this->getSuccessResponse(); }
/** * Save harvested records to disk and track the end date. * * @param object $records SimpleXML records. * * @return void */ protected function processRecords($records) { Console::writeLine('Processing ' . count($records) . " records..."); // Array for tracking successfully harvested IDs: $harvestedIds = array(); // Loop through the records: foreach ($records as $record) { // Die if the record is missing its header: if (empty($record->header)) { throw new \Exception("Unexpected missing record header."); } // Get the ID of the current record: $id = $this->extractID($record); // Save the current record, either as a deleted or as a regular file: $attribs = $record->header->attributes(); if (strtolower($attribs['status']) == 'deleted') { $this->saveDeletedRecord($id); } else { $this->saveRecord($id, $record); $harvestedIds[] = $id; } // If the current record's date is newer than the previous end date, // remember it for future reference: $date = $this->normalizeDate($record->header->datestamp); if ($date && $date > $this->endDate) { $this->endDate = $date; } } // Do we have IDs to log and a log filename? If so, log them: if (!empty($this->harvestedIdLog) && !empty($harvestedIds)) { $file = fopen($this->basePath . $this->harvestedIdLog, 'a'); if (!$file) { throw new \Exception("Problem opening {$this->harvestedIdLog}."); } fputs($file, implode(PHP_EOL, $harvestedIds)); fclose($file); } }
/** * Log a message to the console * * @param string $str message string * * @return void */ protected function logMessage($str) { if ($this->verbose) { Console::writeLine($str); } }
protected function _showLine($line) { $this->console->writeLine($line); }
/** * Merge harvested MARC records into a single <collection> * * @return \Zend\Console\Response * @author Thomas Schwaerzler <*****@*****.**> */ public function mergemarcAction() { $this->checkLocalSetting(); $argv = $this->consoleOpts->getRemainingArgs(); $dir = isset($argv[0]) ? rtrim($argv[0], '/') : ''; if (empty($dir)) { $scriptName = $this->getRequest()->getScriptName(); Console::writeLine('Merge MARC XML files into a single <collection>;'); Console::writeLine('writes to stdout.'); Console::writeLine(''); Console::writeLine('Usage: ' . $scriptName . ' <path_to_directory>'); Console::writeLine('<path_to_directory>: a directory containing MARC XML files to merge'); return $this->getFailureResponse(); } if (!($handle = opendir($dir))) { Console::writeLine("Cannot open directory: {$dir}"); return $this->getFailureResponse(); } Console::writeLine('<collection>'); while (false !== ($file = readdir($handle))) { // Only operate on XML files: if (pathinfo($file, PATHINFO_EXTENSION) === "xml") { // get file content $filePath = $dir . '/' . $file; $fileContent = file_get_contents($filePath); // output content: Console::writeLine("<!-- {$filePath} -->"); Console::write($fileContent); } } Console::writeLine('</collection>'); }
/** * Write a module configuration. * * @param string $configPath Path to write to * @param string $config Configuration array to write * * @return void * @throws \Exception */ protected function writeModuleConfig($configPath, $config) { $generator = FileGenerator::fromArray(['body' => 'return ' . var_export($config, true) . ';']); if (!file_put_contents($configPath, $generator->generate())) { throw new \Exception("Cannot write to {$configPath}"); } Console::writeLine("Successfully updated {$configPath}"); }
/** * Process a language directory. * * @param object $dir Directory object from dir() to process * @param Callable $callback Function to run on all .ini files in $dir * @param bool $showStatus Should we display status messages? * * @return void */ protected function processDirectory($dir, $callback, $showStatus = true) { while ($file = $dir->read()) { // Only process .ini files, and ignore native.ini special case file: if (substr($file, -4) == '.ini' && $file !== 'native.ini') { if ($showStatus) { Console::writeLine("Processing {$file}..."); } $callback($dir->path . '/' . $file); } } }
/** * Convert hash algorithms * Expected parameters: oldmethod:oldkey (or none) newmethod:newkey * * @return \Zend\Console\Response */ public function switchdbhashAction() { // Validate command line arguments: $argv = $this->consoleOpts->getRemainingArgs(); if (count($argv) < 1) { Console::writeLine('Expected parameters: newmethod [newkey]'); return $this->getFailureResponse(); } // Pull existing encryption settings from the configuration: $config = $this->getConfig(); if (!isset($config->Authentication->encrypt_ils_password) || !isset($config->Authentication->ils_encryption_key) || !$config->Authentication->encrypt_ils_password) { $oldhash = 'none'; $oldkey = null; } else { $oldhash = isset($config->Authentication->ils_encryption_algo) ? $config->Authentication->ils_encryption_algo : 'blowfish'; $oldkey = $config->Authentication->ils_encryption_key; } // Pull new encryption settings from arguments: $newhash = $argv[0]; $newkey = isset($argv[1]) ? $argv[1] : $oldkey; // No key specified AND no key on file = fatal error: if ($newkey === null) { Console::writeLine('Please specify a key as the second parameter.'); return $this->getFailureResponse(); } // If no changes were requested, abort early: if ($oldkey == $newkey && $oldhash == $newhash) { Console::writeLine('No changes requested -- no action needed.'); return $this->getSuccessResponse(); } // Initialize Mcrypt first, so we can catch any illegal algorithms before // making any changes: try { if ($oldhash != 'none') { $oldCrypt = new Mcrypt(['algorithm' => $oldhash]); } $newCrypt = new Mcrypt(['algorithm' => $newhash]); } catch (\Exception $e) { Console::writeLine($e->getMessage()); return $this->getFailureResponse(); } // Next update the config file, so if we are unable to write the file, // we don't go ahead and make unwanted changes to the database: $configPath = ConfigLocator::getLocalConfigPath('config.ini', null, true); Console::writeLine("\tUpdating {$configPath}..."); $writer = new ConfigWriter($configPath); $writer->set('Authentication', 'encrypt_ils_password', true); $writer->set('Authentication', 'ils_encryption_algo', $newhash); $writer->set('Authentication', 'ils_encryption_key', $newkey); if (!$writer->save()) { Console::writeLine("\tWrite failed!"); return $this->getFailureResponse(); } // Now do the database rewrite: $userTable = $this->getServiceLocator()->get('VuFind\\DbTablePluginManager')->get('User'); $users = $userTable->select(function ($select) { $select->where->isNotNull('cat_username'); }); Console::writeLine("\tConverting hashes for " . count($users) . ' user(s).'); foreach ($users as $row) { $pass = null; if ($oldhash != 'none' && isset($row['cat_pass_enc'])) { $oldcipher = new BlockCipher($oldCrypt); $oldcipher->setKey($oldkey); $pass = $oldcipher->decrypt($row['cat_pass_enc']); } else { $pass = $row['cat_password']; } $newcipher = new BlockCipher($newCrypt); $newcipher->setKey($newkey); $row['cat_password'] = null; $row['cat_pass_enc'] = $newcipher->encrypt($pass); $row->save(); } // If we got this far, all went well! Console::writeLine("\tFinished."); return $this->getSuccessResponse(); }
/** * Recursively scan the remote index to find dates we can retrieve. * * @param string $start The date to use as the basis for scanning; this date * will NOT be included in results. * * @return void */ protected function scanDates($start) { Console::writeLine("Scanning dates after {$start}..."); // Find all dates AFTER the specified start date try { $result = $this->sru->scan('oai.datestamp="' . $start . '"', 0, 250); } catch (\Exception $e) { $result = false; } if (!empty($result)) { // Parse the response: $result = simplexml_load_string($result); if (!$result) { throw new \Exception("Problem loading XML: {$result}"); } // Extract terms from the response: $namespaces = $result->getDocNamespaces(); $result->registerXPathNamespace('ns', $namespaces['']); $result = $result->xpath('ns:terms/ns:term'); // No terms? We've hit the end of the road! if (!is_array($result)) { return; } // Process all the dates in this batch: foreach ($result as $term) { $date = (string) $term->value; $count = (int) $term->numberOfRecords; $this->processDate($date, $count); } } // Continue scanning with results following the last date encountered // in the loop above: if (isset($date)) { $this->scanDates($date); } }
/** * Command-line tool to delete suppressed records from the index. * * @return void */ public function suppressedAction() { // Setup Solr Connection $this->consoleOpts->addRules(array('authorities' => 'Delete authority records instead of bibliographic records')); $core = $this->consoleOpts->getOption('authorities') ? 'authority' : 'biblio'; $solr = ConnectionManager::connectToIndex('Solr', $core); // Make ILS Connection try { $catalog = $this->getILS(); if ($core == 'authority') { $result = $catalog->getSuppressedAuthorityRecords(); } else { $result = $catalog->getSuppressedRecords(); } } catch (\Exception $e) { Console::writeLine("ILS error -- " . $e->getMessage()); return $this->getFailureResponse(); } // Validate result: if (!is_array($result)) { Console::writeLine("Could not obtain suppressed record list from ILS."); return $this->getFailureResponse(); } else { if (empty($result)) { Console::writeLine("No suppressed records to delete."); return $this->getSuccessResponse(); } } // Get Suppressed Records and Delete from index $status = $solr->deleteRecords($result); if ($status) { // Commit and Optimize $solr->commit(); $solr->optimize(); } else { Console::writeLine("Delete failed."); return $this->getFailureResponse(); } return $this->getSuccessResponse(); }
/** * Normalizer * * @return \Zend\Console\Response */ public function normalizeAction() { // Display help message if parameters missing: $argv = $this->consoleOpts->getRemainingArgs(); if (!isset($argv[0])) { Console::writeLine("Usage: {$_SERVER['argv'][0]} [target]"); Console::writeLine("\ttarget - a file or directory to normalize"); return $this->getFailureResponse(); } $normalizer = new ExtendedIniNormalizer(); $target = $argv[0]; if (is_dir($target)) { $normalizer->normalizeDirectory($target); } else { if (is_file($target)) { $normalizer->normalizeFile($target); } else { Console::writeLine("{$target} does not exist."); return $this->getFailureResponse(); } } return $this->getSuccessResponse(); }
/** * Harvest OAI-PMH records. * * @return void */ public function harvestoaiAction() { $this->checkLocalSetting(); // Read Config files $configFile = ConfigReader::getConfigPath('oai.ini', 'harvest'); $oaiSettings = @parse_ini_file($configFile, true); if (empty($oaiSettings)) { Console::writeLine("Please add OAI-PMH settings to oai.ini."); return $this->getFailureResponse(); } // If first command line parameter is set, see if we can limit to just the // specified OAI harvester: $argv = $this->consoleOpts->getRemainingArgs(); if (isset($argv[0])) { if (isset($oaiSettings[$argv[0]])) { $oaiSettings = array($argv[0] => $oaiSettings[$argv[0]]); } else { Console::writeLine("Could not load settings for {$argv[0]}."); return $this->getFailureResponse(); } } // Loop through all the settings and perform harvests: $processed = 0; foreach ($oaiSettings as $target => $settings) { if (!empty($target) && !empty($settings)) { Console::writeLine("Processing {$target}..."); try { $harvest = new OAI($target, $settings); $harvest->launch(); } catch (\Exception $e) { Console::writeLine($e->getMessage()); return $this->getFailureResponse(); } $processed++; } } // All done. Console::writeLine("Completed without errors -- {$processed} source(s) processed."); return $this->getSuccessResponse(); }