/**
* Update a record table for a set of harvesters and repositories
*
* @param string $table
* @param array $allowedRepositoryIdStrings
* @param array $authGroupIdStrings The ids of the groups to check view authorization for,
* May be one of the following or another group Id string:
* edu.middlebury.agents.everyone
* edu.middlebury.agents.all_agents
* If empty, all assets in the specified repositories will be added regardless of
* their visibility.
*
* @return void
* @access public
* @since 3/9/07
*/
function updateTable($table, $allowedRepositoryIdStrings, $authGroupIdStrings)
{
ArgumentValidator::validate($table, StringValidatorRule::getRule());
ArgumentValidator::validate($allowedRepositoryIdStrings, ArrayValidatorRuleWithRule::getRule(StringValidatorRule::getRule()));
ArgumentValidator::validate($authGroupIdStrings, ArrayValidatorRuleWithRule::getRule(StringValidatorRule::getRule()));
$harmoni = Harmoni::instance();
$config = $harmoni->getAttachedData('OAI_CONFIG');
$repositoryManager = Services::getService('Repository');
$authorizationManager = Services::getService('AuthZ');
$idManager = Services::getService("IdManager");
$dbc = Services::getService("DatabaseManager");
$authGroupIds = array();
foreach ($authGroupIdStrings as $id) {
$authGroupIds[] = $idManager->getId($id);
}
$baseCheckQuery = new SelectQuery();
$baseCheckQuery->addTable('oai_' . $table);
$baseCheckQuery->addColumn('datestamp');
$baseCheckQuery->addColumn('deleted');
$baseUpdateQuery = new UpdateQuery();
$baseUpdateQuery->setTable('oai_' . $table);
$baseUpdateColumns = array('datestamp', 'deleted', 'oai_set', 'dc_title', 'dc_description');
$dcUpdateColumns = array('datestamp', 'deleted', 'oai_set', 'dc_title', 'dc_description', 'dc_creator', 'dc_subject', 'dc_contributor', 'dc_publisher', 'dc_date', 'dc_type', 'dc_format', 'dc_identifier', 'dc_source', 'dc_language', 'dc_relation', 'dc_coverage', 'dc_rights');
$baseInsertQuery = new InsertQuery();
$baseInsertQuery->setTable('oai_' . $table);
$baseInsertColumns = array('datestamp', 'oai_identifier', 'deleted', 'oai_set', 'dc_title', 'dc_description');
$dcInsertColumns = array('datestamp', 'oai_identifier', 'deleted', 'oai_set', 'dc_title', 'dc_description', 'dc_creator', 'dc_subject', 'dc_contributor', 'dc_publisher', 'dc_date', 'dc_type', 'dc_format', 'dc_identifier', 'dc_source', 'dc_language', 'dc_relation', 'dc_coverage', 'dc_rights');
$baseDeleteQuery = new UpdateQuery();
$baseDeleteQuery->setTable('oai_' . $table);
$baseDeleteQuery->addValue('deleted', 'true');
$baseDeleteQuery->addRawValue('datestamp', 'NOW()');
$baseUndeleteQuery = new UpdateQuery();
$baseUndeleteQuery->setTable('oai_' . $table);
$baseUndeleteQuery->addValue('deleted', 'false');
$baseUndeleteQuery->addRawValue('datestamp', 'NOW()');
$forceUpdate = false;
$repositories = $repositoryManager->getRepositories();
$r = 0;
if (count($allowedRepositoryIdStrings)) {
$numR = count($allowedRepositoryIdStrings);
} else {
$numR = $repositories->count();
}
$numUpdates = 0;
$numDeleted = 0;
$message = _('Updating OAI records for repository (%1 of %2) : ');
$message = str_replace('%2', $numR, $message);
$instituteId = $idManager->getId('edu.middlebury.agents.users');
$viewId = $idManager->getId('edu.middlebury.authorization.view');
require_once HARMONI . "/utilities/Timer.class.php";
$timer = new Timer();
$timer->start();
$existingRepositoryIds = array();
while ($repositories->hasNext()) {
$updatesInRepository = 0;
$repository = $repositories->next();
$repositoryId = $repository->getId();
// Only work with allowed repositories
if (count($allowedRepositoryIdStrings) && !in_array($repositoryId->getIdString(), $allowedRepositoryIdStrings)) {
continue;
}
$r++;
$existingRepositoryIds[] = $repositoryId->getIdString();
$assets = $repository->getAssets();
$status = new CLIStatusStars(str_replace('%1', $r, $message) . $repository->getDisplayName());
$status->initializeStatistics($assets->count());
$existingAssetIds = array();
while ($assets->hasNext()) {
$asset = $assets->next();
$assetId = $asset->getId();
$existingAssetIds[] = $assetId->getIdString();
try {
$modificationDate = $asset->getModificationDate();
} catch (UnimplementedException $e) {
$modificationDate = DateAndTime::now();
}
$query = $baseCheckQuery->copy();
$query->addWhereEqual("oai_set", $repositoryId->getIdString());
$query->addWhereEqual("oai_identifier", $assetId->getIdString());
$result = $dbc->query($query, $config->getProperty('OAI_DBID'));
if (!$result->getNumberOfRows()) {
// printpre("Doesn't exist:\t".$asset->getDisplayName()."");
$query = $baseInsertQuery->copy();
$query->addValue('oai_set', $repositoryId->getIdString());
$query->addValue('oai_identifier', $assetId->getIdString());
} else {
// printpre("Exists:\t".$asset->getDisplayName()."");
if ($modificationDate->isGreaterThan(DateAndTime::fromString($result->field('datestamp'))) || $forceUpdate) {
// printpre("\tUpdating:\t".$asset->getDisplayName());
$query = $baseUpdateQuery->copy();
$query->addWhereEqual("oai_set", $repositoryId->getIdString());
$query->addWhereEqual("oai_identifier", $assetId->getIdString());
} else {
$query = null;
}
}
if ($query) {
$query->addRawValue('datestamp', 'NOW()');
}
$isCurrentlyDeleted = $result->getNumberOfRows() && $result->field('deleted') == 'true' ? true : false;
$result->free();
if (!count($authGroupIds)) {
$isVisible = true;
} else {
$isVisible = false;
try {
foreach ($authGroupIds as $id) {
if ($authorizationManager->isAuthorized($id, $viewId, $assetId)) {
$isVisible = true;
break;
}
}
} catch (UnknownIdException $e) {
$isVisible = true;
}
}
if ($query) {
//Add the data fields
// Deleted
if ($isVisible) {
$query->addValue('deleted', 'false');
} else {
$query->addValue('deleted', 'true');
}
$query->addValue('dc_title', $asset->getDisplayName());
$query->addValue('dc_description', $asset->getDescription());
$this->addDublinCoreValues($asset, $query);
$dbc->query($query, $config->getProperty('OAI_DBID'));
$updatesInRepository++;
$numUpdates++;
} else {
if ($isCurrentlyDeleted && $isVisible) {
$query = $baseUndeleteQuery->copy();
} else {
if (!$isCurrentlyDeleted && !$isVisible) {
$query = $baseDeleteQuery->copy();
} else {
$query = null;
}
}
if ($query) {
$query->addWhereEqual("oai_set", $repositoryId->getIdString());
$query->addWhereEqual("oai_identifier", $assetId->getIdString());
$dbc->query($query, $config->getProperty('OAI_DBID'));
$updatesInRepository++;
$numUpdates++;
}
}
$status->updateStatistics();
}
// Update any missing assets as deleted
$query = $baseDeleteQuery->copy();
$query->addWhereEqual("oai_set", $repositoryId->getIdString());
if (count($existingAssetIds)) {
$query->addWhereEqual("deleted", "false");
$query->addWhereNotIn("oai_identifier", $existingAssetIds);
}
$result = $dbc->query($query, $config->getProperty('OAI_DBID'));
if ($result->getNumberOfRows()) {
$updatesInRepository = $updatesInRepository + $result->getNumberOfRows();
$numUpdates = $numUpdates + $result->getNumberOfRows();
}
print OAI_UPDATE_OUTPUT_HTML ? "<pre>" : "\n";
print "Elapsed Time:\t";
$timer->end();
printf("%1.2f", $timer->printTime());
print " seconds";
print OAI_UPDATE_OUTPUT_HTML ? "</pre>" : "";
print OAI_UPDATE_OUTPUT_HTML ? "<pre>" : "\n";
print "Updates: " . $updatesInRepository;
print OAI_UPDATE_OUTPUT_HTML ? "</pre>" : "\n";
}
// Update any missing repositories as deleted
$query = $baseDeleteQuery->copy();
$query->addWhereEqual("deleted", "false");
if (count($existingRepositoryIds)) {
$query->addWhereNotIn("oai_set", $existingRepositoryIds);
}
$result = $dbc->query($query, $config->getProperty('OAI_DBID'));
if ($result->getNumberOfRows()) {
$updatesInRepository = $updatesInRepository + $result->getNumberOfRows();
$numUpdates = $numUpdates + $result->getNumberOfRows();
}
print OAI_UPDATE_OUTPUT_HTML ? "<pre>" : "\n";
print "Total Updates:\t" . $numUpdates;
print OAI_UPDATE_OUTPUT_HTML ? "</pre>" : "\n";
}
/**
* Set the media library quota to be the default
*
* @return void
* @access public
* @since 3/20/08
*/
public function useDefaultMediaQuota()
{
$this->mediaQuota = self::$defaultMediaQuota;
$this->recordInDB();
$query = new UpdateQuery();
$query->setTable('segue_slot');
$query->addWhereEqual('shortname', $this->getShortname());
$query->addRawValue('media_quota', 'NULL');
$dbc = Services::getService('DBHandler');
$dbc->query($query, IMPORTER_CONNECTION);
}
/**
* Update the value for this Part.
*
* @param object mixed $value (original type: java.io.Serializable)
*
* @throws object RepositoryException An exception with one of
* the following messages defined in
* org.osid.repository.RepositoryException may be thrown: {@link
* org.osid.repository.RepositoryException#OPERATION_FAILED
* OPERATION_FAILED}, {@link
* org.osid.repository.RepositoryException#PERMISSION_DENIED
* PERMISSION_DENIED}, {@link
* org.osid.repository.RepositoryException#CONFIGURATION_ERROR
* CONFIGURATION_ERROR}, {@link
* org.osid.repository.RepositoryException#UNIMPLEMENTED
* UNIMPLEMENTED}, {@link
* org.osid.repository.RepositoryException#NULL_ARGUMENT
* NULL_ARGUMENT}
*
* @access public
*/
function updateValue($value)
{
if (!is_null($value)) {
ArgumentValidator::validate($value, StringValidatorRule::getRule());
}
// Store the name in the object in case its asked for again.
if (is_null($value)) {
$this->_name = '';
} else {
$this->_name = $value;
}
// then write it to the database.
$dbHandler = Services::getService("DatabaseManager");
// Check to see if the name is in the database
$query = new SelectQuery();
$query->addTable("dr_file");
$query->addColumn("COUNT(*) as count");
$query->addWhere("id = '" . $this->_recordId->getIdString() . "'");
$result = $dbHandler->query($query, $this->_configuration->getProperty("database_index"));
// If it already exists, use an update query.
if ($result->field("count") > 0) {
$query = new UpdateQuery();
$query->setTable("dr_file");
if (is_null($value)) {
$query->addRawValue("filename", "NULL");
} else {
$query->addValue("filename", $this->_name);
}
$query->addWhere("id = '" . $this->_recordId->getIdString() . "'");
} else {
$query = new InsertQuery();
$query->setTable("dr_file");
$query->addValue("id", $this->_recordId->getIdString());
if (is_null($value)) {
$query->addRawValue("filename", "NULL");
} else {
$query->addValue("filename", $this->_name);
}
}
$result->free();
// run the query
$dbHandler->query($query, $this->_configuration->getProperty("database_index"));
$this->_asset->updateModificationDate();
}
/**
* Record a visit in the database
*
* @param string $slotname
* @return void
* @access protected
* @since 9/22/08
*/
protected function _recordVisit($slotname)
{
$dbc = Services::getService('DatabaseManager');
// First try running an update query, since most will be updates
$query = new UpdateQuery();
$query->setTable('segue_accesslog');
$query->addRawValue('tstamp', 'NOW()');
$query->addWhereEqual('agent_id', $this->_getCurrentAgentId());
$query->addWhereEqual('fk_slotname', $slotname);
$result = $dbc->query($query, IMPORTER_CONNECTION);
// If no rows were updated, insert a new one for this user/slot
if (!$result->getNumberOfRows()) {
$query = new InsertQuery();
$query->setTable('segue_accesslog');
$query->addRawValue('tstamp', 'NOW()');
$query->addValue('agent_id', $this->_getCurrentAgentId());
$query->addValue('fk_slotname', $slotname);
try {
$dbc->query($query, IMPORTER_CONNECTION);
} catch (DuplicateKeyDatabaseException $e) {
// multiple requests may colide, just ignore.
}
}
}
/**
* Execute
*
* @return void
* @access public
* @since 3/26/08
*/
public function execute()
{
if (!$this->isAuthorizedToExecute()) {
throw new PermissionDeniedException('This command can only be run by admins or from the command-line.');
}
header("Content-Type: text/plain");
if (RequestContext::value('help') || RequestContext::value('h') || RequestContext::value('?')) {
throw new HelpRequestedException($this->usage);
}
$outDir = RequestContext::value('d');
if (empty($outDir)) {
throw new InvalidArgumentException("An output directory must be specified.\n\n" . $this->usage);
}
if (!is_dir($outDir) || !is_writable($outDir)) {
throw new InvalidArgumentException("The output directory doesn't exist or is not writeable.\n\n" . $this->usage);
}
foreach (SlotAbstract::getLocationCategories() as $category) {
$baseUrl = SiteDispatcher::getBaseUrlForLocationCategory($category);
if (!preg_match('/^https?:\\/\\/.+/', $baseUrl)) {
throw new ConfigurationErrorException('Please set a base URL for the \'' . $category . '\' category with SiteDispatcher::setBaseUrlForLocationCategory($category, $url); in config/slots.conf.php');
}
}
while (ob_get_level()) {
ob_end_flush();
}
flush();
/*********************************************************
* Check for a running export
*********************************************************/
$dbc = Services::getService('DatabaseManager');
$query = new SelectQuery();
$query->addColumn('slot');
$query->addColumn('pid');
$query->addTable('site_export_queue');
$query->addWhereNotEqual('pid', 0);
$result = $dbc->query($query);
// If we are exporting, check the status of the export process
if ($result->hasMoreRows()) {
// Don't start a new export if one is running.
if ($this->isRunning($result->field('pid'))) {
print "An export is already running\n";
exit;
} else {
$query = new UpdateQuery();
$query->setTable('site_export_queue');
$query->addValue('status', 'DIED');
$query->addRawValue('pid', 'NULL');
$query->addValue('info', 'Process ' . $result->field('pid') . ' has died.');
$query->addWhereEqual('slot', $result->field('slot'));
$query->addWhereEqual('pid', $result->field('pid'));
$dbc->query($query);
}
}
/*********************************************************
* If there aren't any other exports happening, run our export
*********************************************************/
// Find the next slot to update
$query = new SelectQuery();
$query->addColumn('slot');
$query->addTable('site_export_queue', NO_JOIN, '', 'q');
$query->addTable('segue_slot', INNER_JOIN, 'q.slot = s.shortname', 's');
$query->addWhereNull('pid');
$query->addWhereNull('status');
$query->addWhereNull('alias_target');
$query->addWhereNotEqual('site_id', '');
$query->addOrderBy('priority', DESCENDING);
$query->addOrderBy('slot', ASCENDING);
$result = $dbc->query($query);
// Exit if there is nothing to do.
if (!$result->hasMoreRows()) {
print "The queue is empty\n";
exit;
}
$slot = $result->field('slot');
$slotMgr = SlotManager::instance();
$slotObj = $slotMgr->getSlotByShortname($slot);
$baseUrl = SiteDispatcher::getBaseUrlForLocationCategory($slotObj->getLocationCategory());
// Mark that we are running
$query = new UpdateQuery();
$query->setTable('site_export_queue');
$query->addValue('pid', strval(getmypid()));
$query->addWhereEqual('slot', $slot);
$dbc->query($query);
// Run the export
$start = microtime(true);
try {
$exportDirname = $slot . "-html";
$exportDir = $outDir . "/" . $exportDirname;
$archivePath = $outDir . '/' . $exportDirname . ".zip";
if (file_exists($exportDir)) {
$this->deleteRecursive($exportDir);
}
mkdir($exportDir);
if (file_exists($archivePath)) {
unlink($archivePath);
}
// Set the user to be an admin.
$idMgr = Services::getService("Id");
$authType = new Type("Authentication", "edu.middlebury.harmoni", "Harmoni DB");
$_SESSION['__AuthenticatedAgents']['Authentication::edu.middlebury.harmoni::Harmoni DB'] = $idMgr->getId('17008');
$authZ = Services::getService("AuthZ");
$isAuthorizedCache = $authZ->getIsAuthorizedCache();
$isAuthorizedCache->dirtyUser();
// Close the session. If we don't, a lock on the session file will
// cause the request initiated via wget to hang.
session_write_close();
// Do the export
$urlParts = parse_url($baseUrl);
$urlPrefix = rtrim($urlParts['path'], '/');
$include = array($urlPrefix . '/gui2', $urlPrefix . '/images', $urlPrefix . '/javascript', $urlPrefix . '/polyphony', $urlPrefix . '/repository', $urlPrefix . '/plugin_manager', $urlPrefix . '/rss', $urlPrefix . '/dataport/html/site/' . $slot);
if (defined('WGET_PATH')) {
$wget = WGET_PATH;
} else {
$wget = 'wget';
}
if (defined('WGET_OPTIONS')) {
$wgetOptions = WGET_OPTIONS;
} else {
$wgetOptions = '';
}
$command = $wget . " " . $wgetOptions . " -r --page-requisites --html-extension --convert-links --no-directories -e robots=off " . "--directory-prefix=" . escapeshellarg($exportDir . '/content') . " " . "--include=" . escapeshellarg(implode(',', $include)) . " " . "--header=" . escapeshellarg("Cookie: " . session_name() . "=" . session_id()) . " " . escapeshellarg($baseUrl . '/dataport/html/site/' . $slot);
print "Cookie: " . session_name() . "=" . session_id() . "\n";
// throw new Exception($command);
exec($command, $output, $exitCode);
if ($exitCode) {
throw new Exception('Wget Failed. ' . implode("\n", $output));
}
// Copy the main HTML file to index.html
copy($exportDir . '/content/' . $slot . '.html', $exportDir . '/content/index.html');
// Copy the index.html file up a level to make it easy to find
file_put_contents($exportDir . '/index.html', preg_replace('/(src|href)=([\'"])([^\'"\\/]+)([\'"])/', '$1=$2content/$3$4', file_get_contents($exportDir . '/content/index.html')));
// Zip up the result
$archive = new ZipArchive();
if ($archive->open($archivePath, ZIPARCHIVE::CREATE) !== TRUE) {
throw new Exception("Could not create zip archive.");
}
$this->addDirectoryToZip($archive, $exportDir, $exportDirname);
$archive->close();
// Remove the directory
$this->deleteRecursive($exportDir);
// Mark our success
$query = new UpdateQuery();
$query->setTable('site_export_queue');
$query->addRawValue('pid', 'NULL');
$query->addValue('status', 'SUCCESS');
$query->addValue('running_time', strval(round(microtime(true) - $start, 2)));
$query->addWhereEqual('slot', $slot);
$dbc->query($query);
} catch (Exception $e) {
$this->deleteRecursive($exportDir);
if (file_exists($archivePath)) {
unlink($archivePath);
}
// Mark our failure
$query = new UpdateQuery();
$query->setTable('site_export_queue');
$query->addRawValue('pid', 'NULL');
$query->addValue('status', 'EXCEPTION');
$query->addValue('info', $e->getMessage());
$query->addValue('running_time', strval(round(microtime(true) - $start, 2)));
$query->addWhereEqual('slot', $slot);
$dbc->query($query);
throw $e;
}
exit;
}
