public function testLicensingInfo()
{
$license = $this->deposit->getLicense();
$this->assertEquals(7, count($license));
$this->assertEquals('Open', $license['publishingMode']);
$this->assertEquals('OA GOOD', $license['openAccessPolicy']);
}
/**
* Load the XML document into a DOM and return it. Errors are appended to
* the $report parameter.
*
* For reasons beyond anyone's apparent control, the export may contain
* invalid UTF-8 characters. If the file cannot be parsed as XML, the
* function will attempt to filter out invalid UTF-8 characters and then
* try to load the XML again.
*
* Other errors in the XML, beyond the bad UTF-8, will not be tolerated.
*
* @return DOMDocument
*
* @param Deposit $deposit
* @param string $filename
* @param string $report
*/
private function loadXml(Deposit $deposit, $filename, &$report)
{
$dom = new DOMDocument();
try {
$dom->load($filename, LIBXML_COMPACT | LIBXML_PARSEHUGE);
} catch (Exception $ex) {
if (strpos($ex->getMessage(), 'Input is not proper UTF-8') === false) {
$deposit->addErrorLog('XML file ' . basename($filename) . ' is not parseable: ' . $ex->getMessage());
$report .= $ex->getMessage();
$report .= "\nCannot validate XML.\n";
return;
}
// The XML files can be arbitrarily large, so stream them, filter
// the stream, and write to disk. The result may not fit in memory.
$filteredFilename = "{$filename}-filtered.xml";
$in = fopen($filename, 'rb');
$out = fopen($filteredFilename, 'wb');
$blockSize = 64 * 1024;
// 64k blocks
$changes = 0;
while ($buffer = fread($in, $blockSize)) {
$filtered = iconv('UTF-8', 'UTF-8//IGNORE', $buffer);
$changes += strlen($buffer) - strlen($filtered);
fwrite($out, $filtered);
}
$report .= basename($filename) . " contains {$changes} invalid UTF-8 characters, which have been removed with " . ICONV_IMPL . ' version ' . ICONV_VERSION . ' in PHP ' . PHP_VERSION . "\n";
$report .= basename($filteredFilename) . " will be validated.\n";
$dom->load($filteredFilename, LIBXML_COMPACT | LIBXML_PARSEHUGE);
}
return $dom;
}
public function testGetSizeMultiple()
{
$d1 = new Deposit();
$d1->setPackageSize(1234);
$this->auContainer->addDeposit($d1);
$d2 = new Deposit();
$d2->setPackageSize(4321);
$this->auContainer->addDeposit($d2);
$this->assertEquals(5555, $this->auContainer->getSize());
}
public function testGetCompletedDeposits()
{
$d1 = new Deposit();
$d1->setState('completed');
$this->journal->addDeposit($d1);
$d2 = new Deposit();
$this->journal->addDeposit($d2);
$completed = $this->journal->getCompletedDeposits();
$this->assertInstanceOf('Doctrine\\Common\\Collections\\ArrayCollection', $completed);
$this->assertCount(1, $completed);
}
/**
* Add the metadata from the database to the bag-info.txt file.
*
* @param BagIt $bag
* @param Deposit $deposit
*/
protected function addMetadata(BagIt $bag, Deposit $deposit)
{
$bag->bagInfoData = array();
// @todo this is very very bad. Once BagItPHP is updated it should be $bag->clearAllBagInfo();
$bag->setBagInfoData('External-Identifier', $deposit->getDepositUuid());
$bag->setBagInfoData('PKP-PLN-Deposit-UUID', $deposit->getDepositUuid());
$bag->setBagInfoData('PKP-PLN-Deposit-Received', $deposit->getReceived()->format('c'));
$bag->setBagInfoData('PKP-PLN-Deposit-Volume', $deposit->getVolume());
$bag->setBagInfoData('PKP-PLN-Deposit-Issue', $deposit->getIssue());
$bag->setBagInfoData('PKP-PLN-Deposit-PubDate', $deposit->getPubDate()->format('c'));
$journal = $deposit->getJournal();
$bag->setBagInfoData('PKP-PLN-Journal-UUID', $journal->getUuid());
$bag->setBagInfoData('PKP-PLN-Journal-Title', $journal->getTitle());
$bag->setBagInfoData('PKP-PLN-Journal-ISSN', $journal->getIssn());
$bag->setBagInfoData('PKP-PLN-Journal-URL', $journal->getUrl());
$bag->setBagInfoData('PKP-PLN-Journal-Email', $journal->getEmail());
$bag->setBagInfoData('PKP-PLN-Publisher-Name', $journal->getPublisherName());
$bag->setBagInfoData('PKP-PLN-Publisher-URL', $journal->getPublisherUrl());
foreach ($deposit->getLicense() as $key => $value) {
$bag->setBagInfoData('PKP-PLN-' . $key, $value);
}
}
/**
* Get the path to a processed, staged, bag.
*
* @param Deposit $deposit
*
* @return type
*/
public final function getStagingBagPath(Deposit $deposit)
{
$path = $this->getStagingDir($deposit->getJournal());
return $path . '/' . $deposit->getDepositUuid() . '.zip';
}
/**
* Fetch one deposit from LOCKSSOMatic.
*
* @param Deposit $deposit
* @param string $href
*/
public function fetch(Deposit $deposit, $href)
{
$client = $this->getHttpClient();
$filepath = $this->filePaths->getRestoreDir($deposit->getJournal()) . '/' . basename($href);
$this->logger->notice("Saving {$deposit->getJournal()->getTitle()} vol. {$deposit->getVolume()} no. {$deposit->getIssue()} to {$filepath}");
try {
$client->get($href, array('allow_redirects' => false, 'decode_content' => false, 'save_to' => $filepath));
$hash = strtoupper(hash_file($deposit->getPackageChecksumType(), $filepath));
if ($hash !== $deposit->getPackageChecksumValue()) {
$this->logger->warning("Package checksum failed. Expected {$deposit->getPackageChecksumValue()} but got {$hash}");
}
} catch (Exception $ex) {
$this->logger->error($ex->getMessage());
}
}
/**
* Process one deposit. Fetch the data and write it to the file system.
* Updates the deposit status.
*
* @param Deposit $deposit
*
* @return type
*/
protected function processDeposit(Deposit $deposit)
{
$this->logger->notice("harvest - {$deposit->getDepositUuid()}");
$this->checkSize($deposit);
$response = $this->fetchDeposit($deposit->getUrl(), $deposit->getSize());
$deposit->setFileType($response->getHeader('Content-Type'));
$filePath = $this->filePaths->getHarvestFile($deposit);
return $this->writeDeposit($filePath, $response);
}
/**
* Process one deposit. Fetch the data and write it to the file system.
* Updates the deposit status.
*
* @param Deposit $deposit
*
* @return type
*/
protected function processDeposit(Deposit $deposit)
{
$this->logger->notice("Sending deposit {$deposit->getDepositUuid()}");
return $this->client->createDeposit($deposit);
}
/**
* {@inheritDoc}
*/
protected function doLoad(ObjectManager $manager)
{
$d0 = new Deposit();
$d0->setAction('add');
$d0->setChecksumType('SHA1');
$d0->setChecksumValue('b4b0bfbaadf33f8e678ad9cd0c1da2c9a6e7d204');
$d0->setDepositUuid('d38e7ecb-7d7e-408d-94b0-b00d434fdbd2');
$d0->setDepositReceipt('http://example.com/path/to/receipt');
$d0->setFileType('application/zip');
$d0->setIssue(2);
$d0->setJournal($this->getReference('journal'));
$d0->setPubDate(new DateTime());
$d0->setSize(100);
$d0->setState('depositedByJournal');
$d0->setUrl('http://journal.example.com/path/to/deposit');
$d0->setVolume(1);
$manager->persist($d0);
$d1 = new Deposit();
$d1->setAction('add');
$d1->setChecksumType('SHA1');
$d1->setChecksumValue('f1d2d2f924e986ac86fdf7b36c94bcdf32beec15');
$d1->setDepositUuid('578205CB-0947-4CD3-A384-CDF186F5E86B');
$d1->setDepositReceipt('http://example2.com/path/to/receipt');
$d1->setFileType('application/zip');
$d1->setIssue(4);
$d1->setJournal($this->getReference('journal'));
$d1->setPubDate(new DateTime());
$d1->setSize(1000);
$d1->setState('harvested');
$d1->setUrl('http://journal.example2.com/path/to/deposit');
$d1->setVolume(2);
$manager->persist($d1);
$manager->flush();
$this->setReference('deposit', $d0);
}
/**
* Build a deposit from XML.
*
* @param Journal $journal
* @param SimpleXMLElement $xml
* @param string $action
*
* @return Deposit
*/
public function fromXml(Journal $journal, SimpleXMLElement $xml)
{
$id = $this->getXmlValue($xml, '//atom:id');
$deposit_uuid = strtoupper(substr($id, 9, 36));
$deposit = $this->em->getRepository('AppBundle:Deposit')->findOneBy(array('depositUuid' => $deposit_uuid));
$action = 'edit';
if (!$deposit) {
$action = 'add';
$deposit = new Deposit();
}
$deposit->setAction($action);
$deposit->setState('depositedByJournal');
$deposit->setChecksumType($this->getXmlValue($xml, 'pkp:content/@checksumType'));
$deposit->setChecksumValue($this->getXmlValue($xml, 'pkp:content/@checksumValue'));
$deposit->setDepositUuid($deposit_uuid);
$deposit->setFileType('');
$deposit->setIssue($this->getXmlValue($xml, 'pkp:content/@issue'));
$deposit->setVolume($this->getXmlValue($xml, 'pkp:content/@volume'));
$deposit->setPubDate(new DateTime($this->getXmlValue($xml, 'pkp:content/@pubdate')));
$deposit->setJournal($journal);
$deposit->setSize($this->getXmlValue($xml, 'pkp:content/@size'));
$deposit->setUrl(html_entity_decode($this->getXmlValue($xml, 'pkp:content')));
$deposit->setDepositReceipt($this->buildDepositReceiptUrl($deposit));
$this->getLicensingInfo($deposit, $xml);
if ($action === 'add') {
$deposit->addToProcessingLog('Deposit received.');
} else {
$deposit->addToProcessingLog('Deposit edited or reset by journal manager.');
}
$this->em->persist($deposit);
$this->em->flush();
return $deposit;
}
/**
* Process one deposit. Fetch the data and write it to the file system.
* Updates the deposit status, and may remove the processing files if
* LOCKSSOatic reports agreement.
*
* @param Deposit $deposit
*
* @return type
*/
protected function processDeposit(Deposit $deposit, $force = false)
{
if ($deposit->getPlnState() === 'agreement') {
$this->logger->notice($deposit->getDepositUuid());
$this->delFileTree($this->filePaths->getHarvestFile($deposit), $force);
$this->delFileTree($this->filePaths->getProcessingBagPath($deposit), $force);
// $this->delFileTree($this->filePaths->getStagingBagPath($deposit), $force);
}
}
/**
* Process one deposit. Fetch the data and write it to the file system.
* Updates the deposit status, and may remove the processing files if
* LOCKSSOatic reports agreement.
*
* @param Deposit $deposit
*
* @return type
*/
protected function processDeposit(Deposit $deposit)
{
$this->logger->notice("Checking deposit {$deposit->getDepositUuid()}");
$statement = $this->client->statement($deposit);
$status = (string) $statement->xpath('//atom:category[@scheme="http://purl.org/net/sword/terms/state"]/@term')[0];
$this->logger->notice('Deposit is ' . $status);
$deposit->setPlnState($status);
if ($status === 'agreement' && $this->cleanup) {
$this->logger->notice("Deposit complete. Removing processing files for deposit {$deposit->getId()}.");
unlink($this->filePaths->getHarvestFile($deposit));
$this->deltree($this->filePaths->getProcessingBagPath($deposit));
unlink($this->filePaths->getStagingBagPath($deposit));
}
return $status === 'agreement';
}
/**
* Load the XML from a file and return a DOM. Errors are appended to
* the $report string.
*
* @return DOMDocument
*
* @param Deposit $deposit
* @param string $filename
* @param string $report
*/
private function loadXml(Deposit $deposit, $filename, &$report)
{
$dom = new DOMDocument();
try {
$dom->load($filename, LIBXML_COMPACT | LIBXML_PARSEHUGE);
} catch (Exception $ex) {
if (strpos($ex->getMessage(), 'Input is not proper UTF-8') === false) {
$deposit->addErrorLog('XML file ' . basename($filename) . ' is not parseable, and cannot be scanned for viruses: ' . $ex->getMessage());
$report .= $ex->getMessage();
$report .= "\nCannot scan for viruses.\n";
return;
}
$filteredFilename = "{$filename}-filtered.xml";
$report .= basename($filename) . " contains invalid UTF-8 characters and will not be scanned for viruses.\n";
$report .= basename($filteredFilename) . " will be scanned for viruses instead.\n";
$dom->load($filteredFilename, LIBXML_COMPACT | LIBXML_PARSEHUGE);
}
return $dom;
}
public function receipt(Deposit $deposit)
{
$client = $this->getClient();
$receiptRequest = $client->createRequest('GET', $deposit->getDepositReceipt());
$receiptResponse = $client->send($receiptRequest);
$receiptXml = new SimpleXMLElement($receiptResponse->getBody());
$this->namespaces->registerNamespaces($receiptXml);
return $receiptXml;
}
