protected function createJob($label, $data = array())
{
$job = new Opus_Job();
$job->setLabel($label);
$job->setData($data);
$this->jobIds[] = $job->store();
}
/**
* Perfom work.
*
* @param Opus_Job $job Job description and attached data.
* @return array Array of Jobs to be newly created.
*/
public function work(Opus_Job $job)
{
$data = $job->getData(true);
$message = $data['message'];
$subject = $data['subject'];
$users = $data['users'];
$from = $this->_getFrom();
$fromName = $this->_getFromName();
if (!is_null($users) and !is_array($users)) {
$users = array($users);
}
$recipient = array();
if ($this->lookupRecipients) {
$this->_logger->debug(__CLASS__ . ': Resolving mail addresses for users = {"' . implode('", "', $users) . '"}');
$recipient = $this->getRecipients($users);
} else {
$recipient = $users;
}
// if (empty($recipient)) {
// $this->_logger->info(__CLASS__ . ': No recipients avaiable. Mail canceled.');
// return true;
// }
$mailSendMail = new Opus_Mail_SendMail();
$this->_logger->info(__CLASS__ . ': Sending notification email...');
$this->_logger->debug(__CLASS__ . ': sender: ' . $from);
$mailSendMail->sendMail($from, $fromName, $subject, $message, $recipient);
return true;
}
/**
* Perfom work.
*
* @param Opus_Job $job Job description and attached data.
* @return array Array of Jobs to be newly created.
*/
public function work(Opus_Job $job)
{
if ($job->getLabel() != $this->getActivationLabel()) {
throw new Opus_Job_Worker_InvalidJobException($job->getLabel() . " is not a suitable job for this worker.");
}
$data = $job->getData();
if (!(is_object($data) && isset($data->xml) && !is_null($data->xml))) {
throw new Opus_Job_Worker_InvalidJobException("Incomplete or missing data.");
}
if (null !== $this->_logger) {
$this->_logger->debug("Importing Metadata:\n" . $data->xml);
}
$importer = new Opus_Util_MetadataImport($data->xml);
$importer->run();
}
/**
* Load all published documents from database and check consistency.
* A document is considered as inconsistent, if
*
* - it exists in database, but does not exist in Solr index
* - it exists in Solr index, but does not exist in database or exists
* but with server_state != published
* - it exists both in database and Solr index, but server_date_modified
* timestamps do not coincide
*
* @param Opus_Job $job Job description and attached data.
* @return void
*/
public function work(Opus_Job $job)
{
// make sure we have the right job
if ($job->getLabel() != $this->getActivationLabel()) {
throw new Opus_Job_Worker_InvalidJobException($job->getLabel() . " is not a suitable job for this worker.");
}
$lockFile = $this->logfilePath . '.lock';
if (file_exists($lockFile)) {
unlink($lockFile);
}
touch($lockFile);
$consistencyChecker = new Opus_Util_ConsistencyCheck($this->_logger);
$consistencyChecker->run();
unlink($lockFile);
}
public function testJobFailsIfInvalidTask()
{
$this->createJob(Opus_Job_Worker_IndexOpusDocument::LABEL, array('documentId' => $this->document->getId(), 'task' => 'do-the-unexpected'));
$this->executeScript('cron-solr-update.php');
$allJobs = Opus_Job::getByLabels(array(Opus_Job_Worker_IndexOpusDocument::LABEL), null, Opus_Job::STATE_FAILED);
$this->assertEquals(1, count($allJobs), 'Expected one failed job in queue');
}
public function testFailSendNotification()
{
$this->createJob(Opus_Job_Worker_MailNotification::LABEL, array('subject' => 'SendNotification Test', 'message' => 'This is a test message generated in ' . __FILE__, 'users' => ''));
$this->executeScript('cron-send-notification.php');
$failedJobs = Opus_Job::getByLabels(array(Opus_Job_Worker_MailNotification::LABEL), null, Opus_Job::STATE_FAILED);
$this->assertEquals(1, count($failedJobs), 'Expected one failed job in queue');
}
public function detailAction()
{
$this->view->state = $this->_request->getParam('state');
$this->view->label = $this->_request->getParam('label');
if (empty($this->view->state) || empty($this->view->label)) {
throw new Application_Exception('Invalid arguments');
}
$this->view->jobs = Opus_Job::getByLabels(array($this->view->label), null, $this->view->state);
}
protected function tearDown()
{
$testJobs = Opus_Job::getAll($this->jobIds);
foreach ($testJobs as $job) {
$job->delete();
}
Zend_Registry::set('Zend_Config', $this->__configBackup);
parent::tearDown();
}
/**
* TODO fix for Solr Update
*/
public function testJobSuccessWithInconsistency()
{
$service = Opus_Search_Service::selectIndexingService(null, 'solr');
$service->removeAllDocumentsFromIndex();
$this->createJob(Opus_Job_Worker_ConsistencyCheck::LABEL);
$this->executeScript('cron-check-consistency.php');
$allJobs = Opus_Job::getByLabels(array(Opus_Job_Worker_ConsistencyCheck::LABEL), null, Opus_Job::STATE_UNDEFINED);
$this->assertTrue(empty($allJobs), 'Expected no more jobs in queue: found ' . count($allJobs) . ' jobs');
$failedJobs = Opus_Job::getByLabels(array(Opus_Job_Worker_ConsistencyCheck::LABEL), null, Opus_Job::STATE_FAILED);
$this->assertTrue(empty($failedJobs), 'Expected no failed jobs in queue: found ' . count($failedJobs) . ' jobs');
$logPath = parent::$scriptPath . '/../../workspace/log/';
$this->assertFileExists($logPath . 'opus_consistency-check.log', 'Logfile opus_consistency-check.log does not exist');
$this->assertFileNotExists($logPath . 'opus_consistency-check.log.lock', 'Lockfile opus_consistency-check.log.lock was not removed');
$publishedDocsCount = $this->getPublishedDocumentCount();
$contents = file_get_contents($logPath . 'opus_consistency-check.log');
$this->assertFalse(strpos($contents, 'checking ' . $publishedDocsCount . ' published documents for consistency.') === false, "Logfile opus_consistency-check.log does not contain 'checking ' . {$publishedDocsCount}\n . ' ...' [{$contents}].");
$this->assertFalse(strpos($contents, 'inconsistency found for document 1: document is in database, but is not in Solr index.') === false, 'Logfile opus_consistency-check.log does not contain "inconsistency found for document 1: ...".');
$this->assertFalse(strpos($contents, 'inconsistency found for document 200: document is in database, but is not in Solr index.') === false, 'Logfile opus_consistency-check.log does not contain "inconsistency found for document 200: ...".');
$this->assertFalse(strpos($contents, $publishedDocsCount . ' inconsistencies were detected: ' . $publishedDocsCount . ' of them were resolved.') === false, 'Logfile opus_consistency-check.log does not contain "' . $publishedDocsCount . ' inconsistencies ...".');
$this->assertFalse(strpos($contents, 'number of updates: ' . $publishedDocsCount) === false, 'Logfile opus_consistency-check.log does not contain "number of updates: ' . $publishedDocsCount . '".');
$this->assertFalse(strpos($contents, 'number of deletions: 0') === false, 'Logfile opus_consistency-check.log does not contain "number of deletions: 0".');
$this->assertFalse(strpos($contents, 'Completed operation after') === false, 'Logfile opus_consistency-check.log does not contain "Completed operation after".');
unlink($logPath . 'opus_consistency-check.log');
}
public function testJobSuccess()
{
$filename = 'test_import_minimal.xml';
$xml = new DOMDocument();
$this->assertTrue($xml->load($this->xmlDir . $filename), 'Could not load xml as DomDocument');
$this->createJob(Opus_Job_Worker_MetadataImport::LABEL, array('xml' => $xml->saveXML()));
$this->executeScript('cron-import-metadata.php');
$allJobs = Opus_Job::getByLabels(array(Opus_Job_Worker_MetadataImport::LABEL), null, Opus_Job::STATE_UNDEFINED);
$this->assertTrue(empty($allJobs), 'Expected no more jobs in queue');
$failedJobs = Opus_Job::getByLabels(array(Opus_Job_Worker_MetadataImport::LABEL), null, Opus_Job::STATE_FAILED);
$this->assertTrue(empty($failedJobs), 'Expected no failed jobs in queue');
$this->documentImported = true;
}
/**
* Helper method to add document to index.
*
* @param Opus_Document $document
* @return void
*/
private function addDocumentToIndex(Opus_Document $document)
{
$log = Zend_Registry::get('Zend_Log');
// create job if asynchronous is set
if (isset($this->config->runjobs->asynchronous) && $this->config->runjobs->asynchronous) {
$log->debug(__METHOD__ . ': ' . 'Adding index job for document ' . $document->getId() . '.');
$job = new Opus_Job();
$job->setLabel(Opus_Job_Worker_IndexOpusDocument::LABEL);
$job->setData(array('documentId' => $document->getId(), 'task' => 'index'));
// skip creating job if equal job already exists
if (true === $job->isUniqueInQueue()) {
$job->store();
} else {
$log->debug(__METHOD__ . ': ' . 'Indexing job for document ' . $document->getId() . ' already exists!');
}
} else {
$log->debug(__METHOD__ . ': ' . 'Index document ' . $document->getId() . '.');
try {
$indexer = new Opus_SolrSearch_Index_Indexer();
$indexer->addDocumentToEntryIndex($document);
$indexer->commit();
} catch (Opus_SolrSearch_Index_Exception $e) {
$log->debug(__METHOD__ . ': ' . 'Indexing document ' . $document->getId() . ' failed: ' . $e->getMessage());
} catch (InvalidArgumentException $e) {
$log->warn(__METHOD__ . ': ' . $e->getMessage());
}
}
}
private function runJobImmediately()
{
$this->assertEquals(1, Opus_Job::getCountForLabel(Opus_Job_Worker_ConsistencyCheck::LABEL));
$jobrunner = new Opus_Job_Runner();
$jobrunner->setLogger(Zend_Registry::get('Zend_Log'));
$worker = new Opus_Job_Worker_ConsistencyCheck();
$jobrunner->registerWorker($worker);
$jobrunner->run();
$this->assertEquals(0, Opus_Job::getCountForLabel(Opus_Job_Worker_ConsistencyCheck::LABEL));
}
public function testCreateWorkerJobIfAsyncEnabled()
{
$this->config->merge(new Zend_Config(array('runjobs' => array('asynchronous' => 1))));
$this->assertEquals(0, Opus_Job::getCount(), 'test data changed.');
$doc = $this->createTestDocument();
$doc->setLanguage("eng");
$title = new Opus_Title();
$title->setValue("Test Document");
$title->setLanguage("eng");
$doc->addTitleMain($title);
$doc->store();
$this->notification->prepareMail($doc, Application_Util_Notification::SUBMISSION, 'http://localhost/foo/1');
$mailJobs = Opus_Job::getByLabels(array(Opus_Job_Worker_MailNotification::LABEL));
$this->assertEquals(1, count($mailJobs), 'Expected 1 mail job');
$jobs = Opus_Job::getAll();
if (!empty($jobs)) {
foreach ($jobs as $job) {
$job->delete();
}
}
}
public function allowConsistencyCheck()
{
return Opus_Job::getCountForLabel(Opus_Job_Worker_ConsistencyCheck::LABEL) == 0;
}
private function scheduleNotification($subject, $message, $recipients)
{
if (empty($recipients)) {
$this->logger->warn("No recipients could be determined for email notification: skip operation");
return;
}
$addressesUsed = array();
foreach ($recipients as $recipient) {
if (!in_array($recipient['address'], $addressesUsed)) {
$job = new Opus_Job();
$job->setLabel(Opus_Job_Worker_MailNotification::LABEL);
$job->setData(array('subject' => $subject, 'message' => $message, 'users' => array($recipient)));
if (isset($this->config->runjobs->asynchronous) && $this->config->runjobs->asynchronous) {
// Queue job (execute asynchronously)
// skip creating job if equal job already exists
if (true === $job->isUniqueInQueue()) {
$job->store();
}
} else {
// Execute job immediately (synchronously)
try {
$mail = new Opus_Job_Worker_MailNotification($this->logger, false);
$mail->work($job);
} catch (Exception $exc) {
$this->logger->err("Email notification failed: " . $exc);
}
}
array_push($addressesUsed, $recipient['address']);
}
}
}
/**
* Load a document from database and optional file(s) and index them,
* or remove document from index (depending on job)
*
* @param Opus_Job $job Job description and attached data.
* @return void
*/
public function work(Opus_Job $job)
{
// make sure we have the right job
if ($job->getLabel() != $this->getActivationLabel()) {
throw new Opus_Job_Worker_InvalidJobException($job->getLabel() . " is not a suitable job for this worker.");
}
$this->_job = $job;
$data = $job->getData();
if (!(is_object($data) && isset($data->documentId) && isset($data->task))) {
throw new Opus_Job_Worker_InvalidJobException("Incomplete or missing data.");
}
if (null !== $this->_logger) {
$this->_logger->info('Indexing document with ID: ' . $data->documentId . '.');
}
// create index document or remove index, depending on task
if ($data->task === 'index') {
$document = new Opus_Document($data->documentId);
$this->getIndex()->addDocumentToEntryIndex($document)->commit();
} else {
if ($data->task === 'remove') {
$this->getIndex()->removeDocumentFromEntryIndexById($data->documentId)->commit();
} else {
throw new Opus_Job_Worker_InvalidJobException("unknown task '{$data->task}'.");
}
}
}
/**
* Execute a job and remove it from the jobs table on success.
*
* @param Opus_Job $job Job description model.
* @return boolean Returns true if a job is consumend false if not
*/
protected function consume(Opus_Job $job)
{
$label = $job->getLabel();
if ($job->getState() !== null) {
return false;
}
if (array_key_exists($label, $this->_workers)) {
$worker = $this->_workers[$label];
if (null !== $this->_logger) {
$this->_logger->info('Processing ' . $label);
}
$job->setState(Opus_Job::STATE_PROCESSING);
$job->store();
try {
$worker->setLogger($this->_logger);
$worker->work($job);
$job->delete();
sleep($this->_delay);
} catch (Exception $ex) {
if (null !== $this->_logger) {
$msg = get_class($worker) . ': ' . $ex->getMessage();
$this->_logger->err($msg);
}
$job->setErrors(json_encode(array('exception' => get_class($ex), 'message' => $ex->getMessage(), 'trace' => $ex->getTraceAsString())));
$job->setState(Opus_Job::STATE_FAILED);
$job->store();
return false;
}
return true;
}
return false;
}
public function testCheckconsistencyActionResult()
{
$this->enableAsyncIndexmaintenanceMode();
$this->assertEquals(0, Opus_Job::getCountForLabel(Opus_Job_Worker_ConsistencyCheck::LABEL), 'missing cleanup of jobs table');
$this->getRequest()->setMethod('POST');
$this->dispatch('/admin/indexmaintenance/checkconsistency');
$this->assertResponseCode(302);
$this->assertResponseLocationHeader($this->getResponse(), '/admin/indexmaintenance');
$this->assertEquals(1, Opus_Job::getCountForLabel(Opus_Job_Worker_ConsistencyCheck::LABEL), 'consistency check job was not stored in database');
/*
* check if job was scheduled for execution
*/
$this->resetResponse();
$this->resetRequest();
$this->dispatch('/admin/indexmaintenance/index');
$this->assertResponseCode(200, 'foo');
$baseUrl = $this->getRequest()->getBaseUrl();
$body = $this->getResponse()->getBody();
$this->assertContains('div class="opprogress"', $body);
$this->assertNotContains("action=\"{$baseUrl}/admin/indexmaintenance/checkconsistency\"", $body);
// TODO $this->assertContains("action=\"$baseUrl/admin/indexmaintenance/checkfulltexts\"", $body);
// TODO $this->assertContains("action=\"$baseUrl/admin/indexmaintenance/optimizeindex\"", $body);
/*
* run job immediately and check for result
*/
$jobrunner = new Opus_Job_Runner();
$jobrunner->setLogger(Zend_Registry::get('Zend_Log'));
$worker = new Opus_Job_Worker_ConsistencyCheck();
$jobrunner->registerWorker($worker);
$jobrunner->run();
$jobs = Opus_Job::getByLabels(array(Opus_Job_Worker_ConsistencyCheck::LABEL));
if (count($jobs) > 0) {
$job = $jobs[0];
$message = 'at least one unexpected job found (Label: \'%s\', State: \'%s\', Data: \'%s\', Errors: \'%s\\, SHA1 Hash: \'%s\')';
$label = $job->getLabel();
$state = $job->getState();
$data = $job->getData();
$errors = $job->getErrors();
$hash = $job->getSha1Id();
$this->fail(sprintf($message, $label, $state, $data, $errors, $hash));
}
$this->assertEquals(0, Opus_Job::getCountForLabel(Opus_Job_Worker_ConsistencyCheck::LABEL), 'consistency check job was not removed from database after execution');
$this->resetResponse();
$this->resetRequest();
$this->dispatch('/admin/indexmaintenance/index');
$this->assertResponseCode(200, 'bar');
$baseUrl = $this->getRequest()->getBaseUrl();
$body = $this->getResponse()->getBody();
$this->assertNotContains('div class="opprogress"', $body);
$this->assertContains('pre class="opoutput"', $body);
$this->assertContains("action=\"{$baseUrl}/admin/indexmaintenance/checkconsistency\"", $body);
// TODO $this->assertContains("action=\"$baseUrl/admin/indexmaintenance/checkfulltexts\"", $body);
// TODO $this->assertContains("action=\"$baseUrl/admin/indexmaintenance/optimizeindex\"", $body);
}
/**
*
* @param Matheon_Model_Document $document
* @param array $recipient
* @return void
*/
private function __sendPublishNotification($document, $recipient)
{
$config = $this->getConfig();
$baseUrlFiles = $this->view->serverUrl() . '/opus4-matheon/files';
$job = new Opus_Job();
$job->setLabel(Opus_Job_Worker_MailNotification::LABEL);
$job->setData(array('subject' => $document->renderPublishMailSubject(), 'message' => $document->renderPublishMailBody($this->view->fullUrl(), $baseUrlFiles), 'users' => $recipient));
//throw new Exception(var_export($job, true));
if (isset($config->runjobs->asynchronous) && $config->runjobs->asynchronous) {
// Queue job (execute asynchronously)
// skip creating job if equal job already exists
if (true === $job->isUniqueInQueue()) {
$job->store();
}
return true;
}
// Execute job immediately (synchronously)
try {
$mail = new Opus_Job_Worker_MailNotification($this->getLogger());
$mail->work($job);
} catch (Exception $exc) {
$this->getLogger()->err($exc);
}
return true;
}
