/**
* Warn the user if VUFIND_LOCAL_DIR is not set.
*
* @return void
*/
protected function checkLocalSetting()
{
if (!getenv('VUFIND_LOCAL_DIR')) {
Console::writeLine("WARNING: The VUFIND_LOCAL_DIR environment variable is not set.");
Console::writeLine("This should point to your local configuration directory (i.e.");
Console::writeLine(realpath(APPLICATION_PATH . '/local') . ").");
Console::writeLine("Without it, inappropriate default settings may be loaded.");
Console::writeLine("");
}
}
/**
* XSLT Import Tool
*
* @return void
*/
public function importXslAction()
{
// Parse switches:
$this->consoleOpts->addRules(array('test-only' => 'Use test mode', 'index-s' => 'Solr index to use'));
$testMode = $this->consoleOpts->getOption('test-only') ? true : false;
$index = $this->consoleOpts->getOption('index');
if (empty($index)) {
$index = 'Solr';
}
// Display help message if parameters missing:
$argv = $this->consoleOpts->getRemainingArgs();
if (!isset($argv[1])) {
Console::writeLine("Usage: import-xsl.php [--test-only] [--index <type>] " . "XML_file properties_file");
Console::writeLine("\tXML_file - source file to index");
Console::writeLine("\tproperties_file - import configuration file");
Console::writeLine("If the optional --test-only flag is set, " . "transformed XML will be displayed");
Console::writeLine("on screen for debugging purposes, " . "but it will not be indexed into VuFind.");
Console::writeLine("");
Console::writeLine("If the optional --index parameter is set, " . "it must be followed by the name of");
Console::writeLine("a class for accessing Solr; it defaults to the " . "standard Solr class, but could");
Console::writeLine("be overridden with, for example, SolrAuth to " . "load authority records.");
Console::writeLine("");
Console::writeLine("Note: See vudl.properties and ojs.properties " . "for configuration examples.");
return $this->getFailureResponse();
}
// Try to import the document if successful:
try {
$importer = new Importer();
$importer->setServiceLocator($this->getServiceLocator());
$importer->save($argv[0], $argv[1], $index, $testMode);
} catch (\Exception $e) {
Console::writeLine("Fatal error: " . $e->getMessage());
return $this->getFailureResponse();
}
if (!$testMode) {
Console::writeLine("Successfully imported {$argv[0]}...");
}
return $this->getSuccessResponse();
}
/**
* Write a string w/newline to the Console.
*
* @param string $str String to write.
*
* @return void
*/
protected function writeLine($str)
{
// Bypass output when testing:
if (defined('VUFIND_PHPUNIT_RUNNING')) {
return;
}
Console::writeLine($str);
}
/**
* Support method for webcrawlAction().
*
* Process a sitemap URL, either harvesting its contents directly or recursively
* reading in child sitemaps.
*
* @param string $url URL of sitemap to read.
* @param bool $verbose Are we in verbose mode?
* @param string $index Solr index to update
* @param bool $testMode Are we in test mode?
*
* @return bool True on success, false on error.
*/
protected function harvestSitemap($url, $verbose = false, $index = 'SolrWeb', $testMode = false)
{
if ($verbose) {
Console::writeLine("Harvesting {$url}...");
}
$retVal = true;
$file = tempnam('/tmp', 'sitemap');
file_put_contents($file, file_get_contents($url));
$xml = simplexml_load_file($file);
if ($xml) {
// Are there any child sitemaps? If so, pull them in:
$results = isset($xml->sitemap) ? $xml->sitemap : [];
foreach ($results as $current) {
if (isset($current->loc)) {
$success = $this->harvestSitemap((string) $current->loc, $verbose, $index, $testMode);
if (!$success) {
$retVal = false;
}
}
}
// Only import the current sitemap if it contains URLs!
if (isset($xml->url)) {
try {
$this->performImport($file, 'sitemap.properties', $index, $testMode);
} catch (\Exception $e) {
if ($verbose) {
Console::writeLine(get_class($e) . ': ' . $e->getMessage());
}
$retVal = false;
}
}
}
unlink($file);
return $retVal;
}
/**
* Abstract delete method.
*
* @param string $table Table to operate on.
* @param string $successString String for reporting success.
* @param string $failString String for reporting failure.
* @param int $minAge Minimum age allowed for expiration (also used
* as default value).
*
* @return mixed
*/
protected function expire($table, $successString, $failString, $minAge = 2)
{
// Get command-line arguments
$argv = $this->consoleOpts->getRemainingArgs();
// Use command line value as expiration age, or default to $minAge.
$daysOld = isset($argv[0]) ? intval($argv[0]) : $minAge;
// Abort if we have an invalid expiration age.
if ($daysOld < 2) {
Console::writeLine(str_replace('%%age%%', $minAge, 'Expiration age must be at least %%age%% days.'));
return $this->getFailureResponse();
}
// Delete the expired searches--this cleans up any junk left in the database
// from old search histories that were not
// caught by the session garbage collector.
$search = $this->getTable($table);
if (!method_exists($search, 'getExpiredQuery')) {
throw new \Exception($table . ' does not support getExpiredQuery()');
}
$query = $search->getExpiredQuery($daysOld);
if (($count = count($search->select($query))) == 0) {
Console::writeLine($failString);
return $this->getSuccessResponse();
}
$search->delete($query);
Console::writeLine(str_replace('%%count%%', $count, $successString));
return $this->getSuccessResponse();
}
/**
* Save harvested records to disk and track the end date.
*
* @param object $records SimpleXML records.
*
* @return void
*/
protected function processRecords($records)
{
Console::writeLine('Processing ' . count($records) . " records...");
// Array for tracking successfully harvested IDs:
$harvestedIds = array();
// Loop through the records:
foreach ($records as $record) {
// Die if the record is missing its header:
if (empty($record->header)) {
throw new \Exception("Unexpected missing record header.");
}
// Get the ID of the current record:
$id = $this->extractID($record);
// Save the current record, either as a deleted or as a regular file:
$attribs = $record->header->attributes();
if (strtolower($attribs['status']) == 'deleted') {
$this->saveDeletedRecord($id);
} else {
$this->saveRecord($id, $record);
$harvestedIds[] = $id;
}
// If the current record's date is newer than the previous end date,
// remember it for future reference:
$date = $this->normalizeDate($record->header->datestamp);
if ($date && $date > $this->endDate) {
$this->endDate = $date;
}
}
// Do we have IDs to log and a log filename? If so, log them:
if (!empty($this->harvestedIdLog) && !empty($harvestedIds)) {
$file = fopen($this->basePath . $this->harvestedIdLog, 'a');
if (!$file) {
throw new \Exception("Problem opening {$this->harvestedIdLog}.");
}
fputs($file, implode(PHP_EOL, $harvestedIds));
fclose($file);
}
}
/**
* Log a message to the console
*
* @param string $str message string
*
* @return void
*/
protected function logMessage($str)
{
if ($this->verbose) {
Console::writeLine($str);
}
}
protected function _showLine($line)
{
$this->console->writeLine($line);
}
/**
* Merge harvested MARC records into a single <collection>
*
* @return \Zend\Console\Response
* @author Thomas Schwaerzler <*****@*****.**>
*/
public function mergemarcAction()
{
$this->checkLocalSetting();
$argv = $this->consoleOpts->getRemainingArgs();
$dir = isset($argv[0]) ? rtrim($argv[0], '/') : '';
if (empty($dir)) {
$scriptName = $this->getRequest()->getScriptName();
Console::writeLine('Merge MARC XML files into a single <collection>;');
Console::writeLine('writes to stdout.');
Console::writeLine('');
Console::writeLine('Usage: ' . $scriptName . ' <path_to_directory>');
Console::writeLine('<path_to_directory>: a directory containing MARC XML files to merge');
return $this->getFailureResponse();
}
if (!($handle = opendir($dir))) {
Console::writeLine("Cannot open directory: {$dir}");
return $this->getFailureResponse();
}
Console::writeLine('<collection>');
while (false !== ($file = readdir($handle))) {
// Only operate on XML files:
if (pathinfo($file, PATHINFO_EXTENSION) === "xml") {
// get file content
$filePath = $dir . '/' . $file;
$fileContent = file_get_contents($filePath);
// output content:
Console::writeLine("<!-- {$filePath} -->");
Console::write($fileContent);
}
}
Console::writeLine('</collection>');
}
/**
* Write a module configuration.
*
* @param string $configPath Path to write to
* @param string $config Configuration array to write
*
* @return void
* @throws \Exception
*/
protected function writeModuleConfig($configPath, $config)
{
$generator = FileGenerator::fromArray(['body' => 'return ' . var_export($config, true) . ';']);
if (!file_put_contents($configPath, $generator->generate())) {
throw new \Exception("Cannot write to {$configPath}");
}
Console::writeLine("Successfully updated {$configPath}");
}
/**
* Process a language directory.
*
* @param object $dir Directory object from dir() to process
* @param Callable $callback Function to run on all .ini files in $dir
* @param bool $showStatus Should we display status messages?
*
* @return void
*/
protected function processDirectory($dir, $callback, $showStatus = true)
{
while ($file = $dir->read()) {
// Only process .ini files, and ignore native.ini special case file:
if (substr($file, -4) == '.ini' && $file !== 'native.ini') {
if ($showStatus) {
Console::writeLine("Processing {$file}...");
}
$callback($dir->path . '/' . $file);
}
}
}
/**
* Convert hash algorithms
* Expected parameters: oldmethod:oldkey (or none) newmethod:newkey
*
* @return \Zend\Console\Response
*/
public function switchdbhashAction()
{
// Validate command line arguments:
$argv = $this->consoleOpts->getRemainingArgs();
if (count($argv) < 1) {
Console::writeLine('Expected parameters: newmethod [newkey]');
return $this->getFailureResponse();
}
// Pull existing encryption settings from the configuration:
$config = $this->getConfig();
if (!isset($config->Authentication->encrypt_ils_password) || !isset($config->Authentication->ils_encryption_key) || !$config->Authentication->encrypt_ils_password) {
$oldhash = 'none';
$oldkey = null;
} else {
$oldhash = isset($config->Authentication->ils_encryption_algo) ? $config->Authentication->ils_encryption_algo : 'blowfish';
$oldkey = $config->Authentication->ils_encryption_key;
}
// Pull new encryption settings from arguments:
$newhash = $argv[0];
$newkey = isset($argv[1]) ? $argv[1] : $oldkey;
// No key specified AND no key on file = fatal error:
if ($newkey === null) {
Console::writeLine('Please specify a key as the second parameter.');
return $this->getFailureResponse();
}
// If no changes were requested, abort early:
if ($oldkey == $newkey && $oldhash == $newhash) {
Console::writeLine('No changes requested -- no action needed.');
return $this->getSuccessResponse();
}
// Initialize Mcrypt first, so we can catch any illegal algorithms before
// making any changes:
try {
if ($oldhash != 'none') {
$oldCrypt = new Mcrypt(['algorithm' => $oldhash]);
}
$newCrypt = new Mcrypt(['algorithm' => $newhash]);
} catch (\Exception $e) {
Console::writeLine($e->getMessage());
return $this->getFailureResponse();
}
// Next update the config file, so if we are unable to write the file,
// we don't go ahead and make unwanted changes to the database:
$configPath = ConfigLocator::getLocalConfigPath('config.ini', null, true);
Console::writeLine("\tUpdating {$configPath}...");
$writer = new ConfigWriter($configPath);
$writer->set('Authentication', 'encrypt_ils_password', true);
$writer->set('Authentication', 'ils_encryption_algo', $newhash);
$writer->set('Authentication', 'ils_encryption_key', $newkey);
if (!$writer->save()) {
Console::writeLine("\tWrite failed!");
return $this->getFailureResponse();
}
// Now do the database rewrite:
$userTable = $this->getServiceLocator()->get('VuFind\\DbTablePluginManager')->get('User');
$users = $userTable->select(function ($select) {
$select->where->isNotNull('cat_username');
});
Console::writeLine("\tConverting hashes for " . count($users) . ' user(s).');
foreach ($users as $row) {
$pass = null;
if ($oldhash != 'none' && isset($row['cat_pass_enc'])) {
$oldcipher = new BlockCipher($oldCrypt);
$oldcipher->setKey($oldkey);
$pass = $oldcipher->decrypt($row['cat_pass_enc']);
} else {
$pass = $row['cat_password'];
}
$newcipher = new BlockCipher($newCrypt);
$newcipher->setKey($newkey);
$row['cat_password'] = null;
$row['cat_pass_enc'] = $newcipher->encrypt($pass);
$row->save();
}
// If we got this far, all went well!
Console::writeLine("\tFinished.");
return $this->getSuccessResponse();
}
/**
* Recursively scan the remote index to find dates we can retrieve.
*
* @param string $start The date to use as the basis for scanning; this date
* will NOT be included in results.
*
* @return void
*/
protected function scanDates($start)
{
Console::writeLine("Scanning dates after {$start}...");
// Find all dates AFTER the specified start date
try {
$result = $this->sru->scan('oai.datestamp="' . $start . '"', 0, 250);
} catch (\Exception $e) {
$result = false;
}
if (!empty($result)) {
// Parse the response:
$result = simplexml_load_string($result);
if (!$result) {
throw new \Exception("Problem loading XML: {$result}");
}
// Extract terms from the response:
$namespaces = $result->getDocNamespaces();
$result->registerXPathNamespace('ns', $namespaces['']);
$result = $result->xpath('ns:terms/ns:term');
// No terms? We've hit the end of the road!
if (!is_array($result)) {
return;
}
// Process all the dates in this batch:
foreach ($result as $term) {
$date = (string) $term->value;
$count = (int) $term->numberOfRecords;
$this->processDate($date, $count);
}
}
// Continue scanning with results following the last date encountered
// in the loop above:
if (isset($date)) {
$this->scanDates($date);
}
}
/**
* Command-line tool to delete suppressed records from the index.
*
* @return void
*/
public function suppressedAction()
{
// Setup Solr Connection
$this->consoleOpts->addRules(array('authorities' => 'Delete authority records instead of bibliographic records'));
$core = $this->consoleOpts->getOption('authorities') ? 'authority' : 'biblio';
$solr = ConnectionManager::connectToIndex('Solr', $core);
// Make ILS Connection
try {
$catalog = $this->getILS();
if ($core == 'authority') {
$result = $catalog->getSuppressedAuthorityRecords();
} else {
$result = $catalog->getSuppressedRecords();
}
} catch (\Exception $e) {
Console::writeLine("ILS error -- " . $e->getMessage());
return $this->getFailureResponse();
}
// Validate result:
if (!is_array($result)) {
Console::writeLine("Could not obtain suppressed record list from ILS.");
return $this->getFailureResponse();
} else {
if (empty($result)) {
Console::writeLine("No suppressed records to delete.");
return $this->getSuccessResponse();
}
}
// Get Suppressed Records and Delete from index
$status = $solr->deleteRecords($result);
if ($status) {
// Commit and Optimize
$solr->commit();
$solr->optimize();
} else {
Console::writeLine("Delete failed.");
return $this->getFailureResponse();
}
return $this->getSuccessResponse();
}
/**
* Normalizer
*
* @return \Zend\Console\Response
*/
public function normalizeAction()
{
// Display help message if parameters missing:
$argv = $this->consoleOpts->getRemainingArgs();
if (!isset($argv[0])) {
Console::writeLine("Usage: {$_SERVER['argv'][0]} [target]");
Console::writeLine("\ttarget - a file or directory to normalize");
return $this->getFailureResponse();
}
$normalizer = new ExtendedIniNormalizer();
$target = $argv[0];
if (is_dir($target)) {
$normalizer->normalizeDirectory($target);
} else {
if (is_file($target)) {
$normalizer->normalizeFile($target);
} else {
Console::writeLine("{$target} does not exist.");
return $this->getFailureResponse();
}
}
return $this->getSuccessResponse();
}
/**
* Harvest OAI-PMH records.
*
* @return void
*/
public function harvestoaiAction()
{
$this->checkLocalSetting();
// Read Config files
$configFile = ConfigReader::getConfigPath('oai.ini', 'harvest');
$oaiSettings = @parse_ini_file($configFile, true);
if (empty($oaiSettings)) {
Console::writeLine("Please add OAI-PMH settings to oai.ini.");
return $this->getFailureResponse();
}
// If first command line parameter is set, see if we can limit to just the
// specified OAI harvester:
$argv = $this->consoleOpts->getRemainingArgs();
if (isset($argv[0])) {
if (isset($oaiSettings[$argv[0]])) {
$oaiSettings = array($argv[0] => $oaiSettings[$argv[0]]);
} else {
Console::writeLine("Could not load settings for {$argv[0]}.");
return $this->getFailureResponse();
}
}
// Loop through all the settings and perform harvests:
$processed = 0;
foreach ($oaiSettings as $target => $settings) {
if (!empty($target) && !empty($settings)) {
Console::writeLine("Processing {$target}...");
try {
$harvest = new OAI($target, $settings);
$harvest->launch();
} catch (\Exception $e) {
Console::writeLine($e->getMessage());
return $this->getFailureResponse();
}
$processed++;
}
}
// All done.
Console::writeLine("Completed without errors -- {$processed} source(s) processed.");
return $this->getSuccessResponse();
}
