public function testIndex()
{
$account = Account::login("searchzen.org", "test");
$c = $account->collections[0];
$indexer = new Indexer($c);
$indexer->start();
}
function usage($message = null)
{
if ($message != null) {
echo "\n*** {$message} ***\n";
}
echo "\nUsage: php makeindex.php [-options] [<csv filename>]";
echo "\n\nwhere options include:";
echo "\n\n";
echo "-h\t\tprint this information\n";
echo "-f\t\tforce indexation\n";
$indexer = new Indexer();
echo "-d <dir>\tindex data files in directory <dir> (defaults to " . $indexer->get_data_root() . ")\n";
echo "\n";
echo "If <csv filename> is present, index only that file; otherwise index all csv files newer than main index.\n\n";
}
function do_main()
{
//registerTypes registers the mime types and populates the needed tables.
$indexer = Indexer::get();
$indexer->registerTypes();
$oTemplating =& KTTemplating::getSingleton();
$oTemplate =& $oTemplating->loadTemplate('ktcore/search2/reporting/managemimetypes');
$aMimeTypes = KTMime::getAllMimeTypesInformation();
$indexer = Indexer::get();
$numExtensions = 0;
$numIndexedExtensions = 0;
foreach ($aMimeTypes as $key => $mimeType) {
$extractorClass = $mimeType['extractor'];
$numExtensions++;
if (empty($extractorClass)) {
continue;
}
$extractor = $indexer->getExtractor($extractorClass);
$aMimeTypes[$key]['extractor'] = $extractor->getDisplayName();
$numIndexedExtensions++;
}
$indexedPercentage = 0;
if ($numExtensions > 0) {
$indexedPercentage = number_format($numIndexedExtensions * 100 / $numExtensions, 2, '.', ',');
}
$oTemplate->setData(array('context' => $this, 'mime_types' => $aMimeTypes, 'numExtensions' => $numExtensions, 'numIndexedExtensions' => $numIndexedExtensions, 'indexedPercentage' => $indexedPercentage));
return $oTemplate;
}
function setup()
{
//Indexer::checkForRegisteredTypes();
$this->indexer = Indexer::get();
$this->path = KT_DIR . '/tests/documentProcessor/dataset/';
$this->tempPath = KT_DIR . '/var/tmp';
}
public static function instance()
{
if (self::$instance == null) {
self::$instance = new Indexer();
}
return self::$instance;
}
/**
* Execute the console command.
*
* @return mixed
*/
public function fire()
{
$watches = array();
$in = inotify_init();
// add watches starting from root directory
$root = Path::fromRelative('');
$this->addWatches($in, $root, $watches);
printf("\nReading for events\n");
while (true) {
$events = inotify_read($in);
foreach ($events as $event) {
$path = $watches[$event['wd']];
$expanded = $this->expandMask($event['mask']);
$eventName = trim(implode(', ', $expanded), ', ');
// if the event has a name attached, then index that
if ($event['name']) {
$newPathName = $path->getPathname() . '/' . $event['name'];
$newPath = new Path($newPathName);
Indexer::index($newPath, 1);
// this may be a new directory, so add a watch to it anyway
if ($newPath->exists() && $newPath->isDir()) {
try {
$wd = inotify_add_watch($in, $newPath->getPathname(), $this->computedMask);
$watches[$wd] = $newPath;
} catch (Exception $e) {
echo 'Caught exception: ', $e->getMessage(), "\n";
}
}
} else {
// event must apply to this directory, so index it, 1 level deep
Indexer::index($path, 1);
}
}
}
}
function do_main()
{
//registerTypes registers the mime types and populates the needed tables.
$indexer = Indexer::get();
$indexer->registerTypes();
$oTemplating =& KTTemplating::getSingleton();
$oTemplating->addLocation('Extractor Information', '/plugins/search2/reporting/templates');
$oTemplate =& $oTemplating->loadTemplate('extractorinfo');
$aExtractorInfo = KTMime::getMimeExtractorInformation();
if (empty($aExtractorInfo)) {
$oTemplate->setData(array('context' => $this, 'extractor_info' => $aExtractorInfo));
return $oTemplate;
}
foreach ($aExtractorInfo as $key => $info) {
$extractorClass = $info['name'];
$extractor = $indexer->getExtractor($extractorClass);
$info['mimeTypes'] = array();
$aMimeTypes = $this->getSupportedMimeTypesDB($extractorClass);
//$extractor->getSupportedMimeTypes();
foreach ($aMimeTypes as $mimeType) {
$sMimeInfo = KTMime::getFriendlyNameAndExtension($mimeType);
$info['mimeTypes'][$mimeType] = array('description' => $sMimeInfo[0]['friendly_name'], 'extensions' => array($sMimeInfo[0]['filetypes']));
$extensions = array();
foreach ($sMimeInfo as $item) {
$extensions[] = $item['filetypes'];
}
$info['mimeTypes'][$mimeType]['extensions'] = implode(', ', $extensions);
}
$aExtractorInfo[$key] = $info;
}
$oTemplate->setData(array('context' => $this, 'extractor_info' => $aExtractorInfo));
return $oTemplate;
}
/**
* @internal
**/
public static function _query($args, $pageIndex, $size, $sortByDate = false)
{
$query = new \Elastica\Query($args);
$query->setFrom($pageIndex * $size);
$query->setSize($size);
$query->setFields(array('id'));
Config::apply_filters('searcher_query', $query);
try {
$index = Indexer::_index(false);
$search = new \Elastica\Search($index->getClient());
$search->addIndex($index);
if (!$query->hasParam('sort')) {
if ($sortByDate) {
$query->addSort(array('post_date' => 'desc'));
} else {
$query->addSort('_score');
}
}
Config::apply_filters('searcher_search', $search, $query);
$results = $search->search($query);
return self::_parseResults($results);
} catch (\Exception $ex) {
error_log($ex);
return null;
}
}
/**
* @brief Process resource actions
* @global type $tool_content
* @global type $id
* @global type $langResourceCourseUnitDeleted
* @global type $langResourceUnitModified
* @global type $course_id
* @global type $course_code
* @return string
*/
function process_actions()
{
global $tool_content, $id, $langResourceCourseUnitDeleted, $langResourceUnitModified, $course_id, $course_code, $webDir;
// update index and refresh course metadata
require_once 'modules/search/indexer.class.php';
require_once 'modules/course_metadata/CourseXML.php';
if (isset($_REQUEST['edit'])) {
$res_id = intval($_GET['edit']);
if ($id = check_admin_unit_resource($res_id)) {
return edit_res($res_id);
}
} elseif (isset($_REQUEST['edit_res_submit'])) {
// edit resource
$res_id = intval($_REQUEST['resource_id']);
if ($id = check_admin_unit_resource($res_id)) {
@($restitle = $_REQUEST['restitle']);
$rescomments = purify($_REQUEST['rescomments']);
$result = Database::get()->query("UPDATE course_weekly_view_activities SET\n title = ?s,\n comments = ?s\n WHERE course_weekly_view_id = ?d AND id = ?d", $restitle, $rescomments, $id, $res_id);
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_UNITRESOURCE, $res_id);
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_COURSE, $course_id);
CourseXMLElement::refreshCourse($course_id, $course_code);
}
$tool_content .= "<div class='alert alert-success'>{$langResourceUnitModified}</div>";
} elseif (isset($_REQUEST['del'])) {
// delete resource from course unit
$res_id = intval($_GET['del']);
if ($id = check_admin_unit_resource($res_id)) {
Database::get()->query("DELETE FROM course_weekly_view_activities WHERE id = ?d", $res_id);
Indexer::queueAsync(Indexer::REQUEST_REMOVE, Indexer::RESOURCE_UNITRESOURCE, $res_id);
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_COURSE, $course_id);
CourseXMLElement::refreshCourse($course_id, $course_code);
$tool_content .= "<div class='alert alert-success'>{$langResourceCourseUnitDeleted}</div>";
}
} elseif (isset($_REQUEST['vis'])) {
// modify visibility in text resources only
$res_id = intval($_REQUEST['vis']);
if ($id = check_admin_unit_resource($res_id)) {
$vis = Database::get()->querySingle("SELECT `visible` FROM course_weekly_view_activities WHERE id = ?d", $res_id)->visible;
$newvis = $vis == 1 ? 0 : 1;
Database::get()->query("UPDATE course_weekly_view_activities SET visible = '{$newvis}' WHERE id = ?d", $res_id);
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_UNITRESOURCE, $res_id);
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_COURSE, $course_id);
CourseXMLElement::refreshCourse($course_id, $course_code);
}
} elseif (isset($_REQUEST['down'])) {
// change order down
$res_id = intval($_REQUEST['down']);
if ($id = check_admin_unit_resource($res_id)) {
move_order('course_weekly_view_activities', 'id', $res_id, 'order', 'down', "course_weekly_view_id={$id}");
}
} elseif (isset($_REQUEST['up'])) {
// change order up
$res_id = intval($_REQUEST['up']);
if ($id = check_admin_unit_resource($res_id)) {
move_order('course_weekly_view_activities', 'id', $res_id, 'order', 'up', "course_weekly_view_id={$id}");
}
}
return '';
}
/**
* Execute the console command.
*
* @return mixed
*/
public function fire()
{
$basePath = Config::get('app.manga_path');
$path = new Path($basePath);
$count = 0;
Indexer::index($path, null, $count);
$this->info(sprintf('Added %s new paths', $count));
}
function checkLucene()
{
$indexer = Indexer::get();
$diagnose = $indexer->diagnose();
if (!is_null($diagnose)) {
$this->addIssue('Document Indexer', $diagnose);
}
}
/**
* Dispatch function
*/
function do_main()
{
$this->aBreadcrumbs[] = array('url' => $_SERVER['PHP_SELF'], 'name' => _kt('Document Indexer Statistics'));
$indexer = Indexer::get();
$stats = $indexer->getIndexStatistics();
$oTemplating =& KTTemplating::getSingleton();
$oTemplate = $oTemplating->loadTemplate('ktcore/search2/lucene_statistics');
$aTemplateData = array('context' => $this, 'stats' => $stats);
return $oTemplate->render($aTemplateData);
}
function reindex()
{
try {
echo Indexer::reindex($_POST['page']);
} catch (\Exception $ex) {
header("HTTP/1.0 500 Internal Server Error");
echo $ex->getMessage();
}
die;
}
function setup()
{
$this->indexer = Indexer::get();
//$this->path = KT_DIR . '/tests/documentProcessor/dataset/';
$tempPath = KT_DIR . '/var/tmp';
$this->targetFile = tempnam($tempPath, 'ktindexer');
file_put_contents($this->targetFile, $this->getContent());
$this->id = 'test_id_1';
$this->title = 'The Critique of Pure Reason';
$this->version = '0.1';
}
function delete_post($post_id)
{
if (is_object($post_id)) {
$post = $post_id;
} else {
$post = get_post($post_id);
}
if ($post == null || !in_array($post->post_type, Config::types())) {
return;
}
Indexer::delete($post);
}
/**
* Dispatch function
*/
function do_main()
{
// Load Templating Engine
$oTemplating =& KTTemplating::getSingleton();
// Set Template to use
$oTemplate = $oTemplating->loadTemplate('ktcore/search2/indexing_status');
// Do a runtime Service Resource Check
$checker = new ResourceChecker();
$checker->check();
// Get Results
$serviceErrors = KTUtil::getSystemSetting('externalResourceIssues');
$serviceErrors = unserialize($serviceErrors);
// Array to Hold Items not working
$this->serviceErrors = array();
// A reference array to use with extractors that depend on the service
$errorReference = array();
if (count($serviceErrors) > 0) {
// Add Service Errors
foreach ($serviceErrors as $error) {
$this->serviceErrors[$error['name']] = array('name' => $error['name'], 'status' => $error['status'], 'help' => IndexingHelp::getHelp($error['name']));
// Create Reference to the array
$errorReference[$error['status']] =& $this->serviceErrors[$error['name']];
}
}
// ---------------------------
// Do a run time check for extractors not working
Indexer::updateIndexStats();
// Get Results
$extractorDiagnosis = KTUtil::getSystemSetting('extractorDiagnostics');
if (!empty($extractorDiagnosis)) {
$extractorDiagnosis = unserialize($extractorDiagnosis);
}
// Create an Array to store errors
$this->extractorErrors = array();
if (count($extractorDiagnosis > 0)) {
foreach ($extractorDiagnosis as $extractor => $props) {
// Check if Extractor fault is due to a service not running
if (array_key_exists($props['diagnosis'], $errorReference)) {
// One service may affect multiple extractors
$errorReference[$props['diagnosis']]['alsoaffects'] = array(array('extractor' => $props['name'], 'affectedtypes' => IndexingHelp::affectedTypes($extractor)));
} else {
// Else list as normal extractor error
$this->extractorErrors[$extractor] = array('name' => $props['name'], 'status' => $props['diagnosis'], 'help' => IndexingHelp::getHelp($extractor), 'affectedtypes' => IndexingHelp::affectedTypes($extractor));
}
}
}
// Create URL to Send to Template
$url = KTUtil::kt_url();
// Prepare Template Data
$aTemplateData = array('context' => $this, 'serviceErrors' => $this->serviceErrors, 'extractorErrors' => $this->extractorErrors, 'url' => $url);
// Send to template and render
return $oTemplate->render($aTemplateData);
}
function do_main()
{
//registerTypes registers the mime types and populates the needed tables.
$indexer = Indexer::get();
$indexer->registerTypes();
$oTemplating =& KTTemplating::getSingleton();
$oTemplate =& $oTemplating->loadTemplate('ktcore/search2/reporting/rescheduledocuments');
if ($_REQUEST['rescheduleValue'] == 'reschedule') {
Indexer::indexAll();
$oTemplate->setData(array('context' => $this, 'rescheduleDone' => true));
return $oTemplate;
}
$oTemplate->setData(array('context' => $this, 'rescheduleDone' => false));
return $oTemplate;
}
function do_main()
{
$folder = $this->oFolder;
$folderid = $folder->getId();
if (Permission::userIsSystemAdministrator()) {
if ($folderid == 1) {
Indexer::indexAll();
} else {
Indexer::indexFolder($folder);
}
}
$full_path = $folder->getFullPath();
$this->addInfoMessage(sprintf(_kt("All documents under the folder '%s' have been scheduled for indexing."), $full_path));
redirect("browse.php?fFolderId={$folderid}");
exit;
}
function enterFunc($nickname){
printlog(1, "-=$nickname=-entered\n");
global $nickList;
global $blacklist;
$nick_array = array_diff(array($nickname), $blacklist);
if( !is_array($nick_array) || !count($nick_array)){
return;
}
$nick_array = Indexer::getNeedNicks( $nick_array );
if( !is_array($nick_array) || !count($nick_array)){
return;
}
$nickList[] = $nickname;
}
/**
* Refresh the hierarchy nodes (departments) that a course belongs to. All previous belonging
* nodes get deleted and then refreshed with the ones given as array arguments.
*
* @param int $id - Id for a given course
* @param array $departments - Array containing the node ids that the given course should belong to
*/
public function refresh($id, $departments)
{
if ($departments != null) {
Database::get()->query("DELETE FROM {$this->departmenttable} WHERE course = ?d", $id);
foreach (array_unique($departments) as $key => $department) {
Database::get()->query("INSERT INTO {$this->departmenttable} (course, department) VALUES (?d,?d)", $id, $department);
}
}
// refresh index
global $webDir;
// required for indexer
require_once 'modules/search/indexer.class.php';
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_COURSE, $id);
// refresh course metadata
require_once 'modules/course_metadata/CourseXML.php';
CourseXMLElement::refreshCourse($id, course_id_to_code($id));
}
function do_main()
{
$doc = $this->oDocument;
$docid = $doc->getId();
if (Permission::userIsSystemAdministrator()) {
$full_path = $doc->getFullPath();
if (Indexer::isDocumentScheduled($docid)) {
Indexer::unqueueDocument($docid);
$this->addInfoMessage(sprintf(_kt("Document '%s' has been removed from the indexing queue."), $full_path));
} else {
Indexer::index($doc, 'A');
$this->addInfoMessage(sprintf(_kt("Document '%s' has been added to the indexing queue."), $full_path));
}
}
redirect("view.php?fDocumentId={$docid}");
exit;
}
function do_main()
{
//registerTypes registers the mime types and populates the needed tables.
$indexer = Indexer::get();
$indexer->registerTypes();
$aPendingDocs = Indexer::getPendingIndexingQueue();
foreach ($aPendingDocs as $key => $doc) {
$extractor = $indexer->getExtractor($doc['extractor']);
if (is_null($extractor)) {
$doc['extractor'] = 'n/a';
continue;
}
$doc['extractor'] = $extractor->getDisplayName();
$aPendingDocs[$key] = $doc;
}
$oTemplating =& KTTemplating::getSingleton();
$oTemplate =& $oTemplating->loadTemplate('ktcore/search2/reporting/pendingdocuments');
$config = KTConfig::getSingleton();
$rootUrl = $config->get('KnowledgeTree/rootUrl');
$oTemplate->setData(array('context' => $this, 'pending_docs' => $aPendingDocs, 'root_url' => $rootUrl));
return $oTemplate;
}
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* You can contact KnowledgeTree Inc., PO Box 7775 #87847, San Francisco,
* California 94120-7775, or email info@knowledgetree.com.
*
* The interactive user interfaces in modified source and object code versions
* of this program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU General Public License version 3.
*
* In accordance with Section 7(b) of the GNU General Public License version 3,
* these Appropriate Legal Notices must retain the display of the "Powered by
* KnowledgeTree" logo and retain the original copyright notice. If the display of the
* logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices
* must display the words "Powered by KnowledgeTree" and retain the original
* copyright notice.
* Contributor( s): ______________________________________
*
*/
/**
* PURPOSE:
*
* This script is to be called periodically to index documents.
*/
chdir(dirname(__FILE__));
require_once realpath('../../../config/dmsDefaults.php');
require_once 'indexing/indexerCore.inc.php';
$indexer = Indexer::get();
$indexer->indexDocuments();
$indexer->updateIndexStats();
exit;
$require_admin = true;
require_once '../../include/baseTheme.php';
require_once 'modules/search/indexer.class.php';
header('Content-Type: application/json; charset=utf-8');
// fetch number of courses waiting in index queue
$n = Database::get()->querySingle("SELECT COUNT(id) AS count FROM idx_queue")->count;
$rem = $n;
if ($n > 0) {
// fetch next waiting course
$cid = Database::get()->querySingle("SELECT course_id FROM idx_queue LIMIT 1")->course_id;
// re-index
$idx = new Indexer();
$idx->removeAllByCourse($cid);
$idx->storeAllByCourse($cid);
set_time_limit(0);
$idx->getIndex()->optimize();
// remove course from queue
Database::get()->query("DELETE FROM idx_queue WHERE course_id = ?d", $cid);
$rem = $n - 1;
}
$data = array(
"remaining" => $rem
);
echo json_encode($data);
INNER JOIN forum_topic ON forum_post.topic_id = forum_topic.id
INNER JOIN forum ON forum.id = forum_topic.forum_id
WHERE forum_post.poster_id = ?d AND forum.course_id = ?d", $author, $course_id);
Database::get()->query("DELETE FROM forum_user_stats WHERE user_id = ?d AND course_id = ?d", $author, $course_id);
if ($forum_user_stats->c != 0) {
Database::get()->query("INSERT INTO forum_user_stats (user_id, num_posts, course_id) VALUES (?d,?d,?d)", $author, $forum_user_stats->c, $course_id);
}
}
Indexer::queueAsync(Indexer::REQUEST_REMOVEBYTOPIC, Indexer::RESOURCE_FORUMPOST, $topic_id);
$number_of_topics = get_total_topics($forum_id);
$num_topics = $number_of_topics - 1;
if ($number_of_topics < 0) {
$num_topics = 0;
}
Database::get()->query("DELETE FROM forum_topic WHERE id = ?d AND forum_id = ?d", $topic_id, $forum_id);
Indexer::queueAsync(Indexer::REQUEST_REMOVE, Indexer::RESOURCE_FORUMTOPIC, $topic_id);
Database::get()->query("UPDATE forum SET num_topics = ?d,
num_posts = num_posts-$number_of_posts
WHERE id = ?d
AND course_id = ?d", $num_topics, $forum_id, $course_id);
Database::get()->query("DELETE FROM forum_notify WHERE topic_id = ?d AND course_id = ?d", $topic_id, $course_id);
Session::Messages($langDeletedMessage, 'alert-success');
redirect_to_home_page("modules/forum/viewforum.php?course=$course_code&forum=$forum_id");
}
// modify topic notification
if (isset($_GET['topicnotify'])) {
if (isset($_GET['topic_id'])) {
$topic_id = intval($_GET['topic_id']);
}
$rows = Database::get()->querySingle("SELECT COUNT(*) AS count FROM forum_notify
$newVisibilityStatus = 0;
$visibilityPath = $_GET['mkInvisibl'];
}
Database::get()->query("UPDATE document SET visible=?d\n WHERE {$group_sql} AND\n path = ?s", $newVisibilityStatus, $visibilityPath);
$r = Database::get()->querySingle("SELECT id FROM document WHERE {$group_sql} AND path = ?s", $visibilityPath);
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_DOCUMENT, $r->id);
Session::Messages($langViMod, 'alert-success');
redirect_to_home_page("modules/document/index.php?course={$course_code}");
}
// Public accessibility commands
if (isset($_GET['public']) || isset($_GET['limited'])) {
$new_public_status = intval(isset($_GET['public']));
$path = isset($_GET['public']) ? $_GET['public'] : $_GET['limited'];
Database::get()->query("UPDATE document SET public = ?d\n WHERE {$group_sql} AND\n path = ?s", $new_public_status, $path);
$r = Database::get()->querySingle("SELECT id FROM document WHERE {$group_sql} AND path = ?s", $path);
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_DOCUMENT, $r->id);
$action_message = "<div class='alert alert-success'>{$langViMod}</div>";
}
}
// teacher only
// Common for teachers and students
// define current directory
// Check if $var is set and return it - if $is_file, then return only dirname part
function pathvar(&$var, $is_file = false)
{
static $found = false;
if ($found) {
return '';
}
if (isset($var)) {
$found = true;
Windows Vista always returns false if we try and check the host and port
so for windows we use the win32 service status checks.
*/
// Check if the calling function requires a return value
$sGiveOutput = isset($argv[1]) && $argv[1] == 'output' ? true : false;
// Check indexed document count
// If the number of indexed documents is greater than the set amount, restart open office
// this clears open office's memory usage
$resetPoint = 50;
// todo: put in config
$count = Indexer::getIndexedDocumentCount();
$restartOO = false;
if ($count > $resetPoint) {
$restartOO = true;
// reset the count
Indexer::updateIndexedDocumentCount(0);
$default->log->debug('Check Open Office Task: Restarting open office.');
}
// First we check the host:port to see if open office is running
$sCheckOO = SearchHelper::checkOpenOfficeAvailablity();
if (empty($sCheckOO) && !$restartOO) {
// If the check returns empty then it is available on that port so we exit
if ($sGiveOutput) {
echo 1;
}
exit;
}
// Open office appears not to be running or requires a restart
if (OS_WINDOWS) {
$OOService = 'ktopenoffice';
$default->log->debug('Check Open Office Task: ' . get_current_user());
/**
* Process a document - extract text and index it
* Refactored from indexDocuments()
*
* @param unknown_type $docinfo
*/
public function processDocument($document, $docinfo)
{
global $default;
static $extractorCache = array();
// increment indexed documents count
Indexer::incrementCount();
// if document is a zero byte file, let's just unqueue and return
if ($document->getFileSize() == 0) {
Indexer::unqueueDocument($docinfo['document_id'], sprintf(_kt("Zero Byte documents do not need to be indexed: %d"), $docinfo['document_id']));
return;
}
$docId = $docinfo['document_id'];
$extension = $docinfo['filetypes'];
$mimeType = $docinfo['mimetypes'];
$extractorClass = $docinfo['extractor'];
$indexDocument = in_array($docinfo['what'], array('A', 'C'));
$indexDiscussion = in_array($docinfo['what'], array('A', 'D'));
$this->indexingHistory = '';
$tempPath = $this->tempPath;
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Indexing docid: %d extension: '%s' mimetype: '%s' extractor: '%s'"), $docId, $extension, $mimeType, $extractorClass), 'debug');
if (empty($extractorClass)) {
/*
if no extractor is found and we don't need to index discussions, then we can remove the item from the queue.
*/
if ($indexDiscussion) {
$indexDocument = false;
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Not indexing docid: %d content because extractor could not be resolve. Still indexing discussion."), $docId), 'info');
} else {
Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"), $docId));
return;
}
} else {
/*
If an extractor is available, we must ensure it is enabled.
*/
if (!$this->isExtractorEnabled($extractorClass)) {
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info');
return;
}
}
if ($this->debug) {
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Processing docid: %d.\n"), $docId), 'info');
}
if ($this->restartCurrentBatch) {
Indexer::unqueueDocument($docId);
Indexer::index($docId, 'A');
return;
}
$filename = $document->getFileName();
if (substr($filename, 0, 1) == '~' || substr($filename, -1) == '~') {
Indexer::unqueueDocument($docId, sprintf(_kt("indexDocuments: Filename for document id %d starts with a tilde (~). This is assumed to be a temporary file. This is ignored."), $docId), 'error');
return;
}
$removeFromQueue = true;
if ($indexDocument) {
if (array_key_exists($extractorClass, $extractorCache)) {
$extractor = $extractorCache[$extractorClass];
} else {
$extractor = $extractorCache[$extractorClass] = $this->getExtractor($extractorClass);
}
if (!$extractor instanceof DocumentExtractor) {
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("indexDocuments: extractor '%s' is not a document extractor class."), $extractorClass), 'error');
return;
}
$version = $document->getMajorVersionNumber() . '.' . $document->getMinorVersionNumber();
$sourceFile = $this->storageManager->temporaryFile($document);
if (empty($sourceFile) || !is_file($sourceFile)) {
Indexer::unqueueDocument($docId, sprintf(_kt("indexDocuments: source file '%s' for document %d does not exist."), $sourceFile, $docId), 'error');
continue;
}
if ($extractor->needsIntermediateSourceFile()) {
//$extension = pathinfo($document->getFileName(), PATHINFO_EXTENSION);
$intermediate = $tempPath . '/' . $docId . '.' . $extension;
$result = @copy($sourceFile, $intermediate);
if ($result === false) {
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Could not create intermediate file from document %d"), $docId), 'error');
// problem. lets try again later. probably permission related. log the issue.
continue;
}
$sourceFile = $intermediate;
}
$extractor->setSourceFile($sourceFile);
$extractor->setMimeType($mimeType);
$extractor->setExtension($extension);
$extractor->setDocument($document);
$extractor->setIndexingStatus(null);
$extractor->setExtractionStatus(null);
$targetFile = tempnam($tempPath, 'ktindexer');
$extractor->setTargetFile($targetFile);
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Extra Info docid: %d Source File: '%s' Target File: '%s'"), $docId, $sourceFile, $targetFile), 'debug');
$this->executeHook($extractor, 'pre_extract');
$this->executeHook($extractor, 'pre_extract', $mimeType);
$removeFromQueue = false;
if ($extractor->extractTextContent()) {
// the extractor may need to create another target file
$targetFile = $extractor->getTargetFile();
$extractor->setExtractionStatus(true);
$this->executeHook($extractor, 'pre_index');
$this->executeHook($extractor, 'pre_index', $mimeType);
$title = $document->getName();
if ($indexDiscussion) {
if (!$this->filterText($targetFile)) {
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"), $docId), 'error');
} else {
$indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
$removeFromQueue = $indexStatus;
if (!$indexStatus) {
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"), $docId), 'error');
}
$extractor->setIndexingStatus($indexStatus);
}
} else {
if (!$this->filterText($targetFile)) {
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"), $docId), 'error');
} else {
$indexStatus = $this->indexDocument($docId, $targetFile, $title, $version);
$removeFromQueue = $indexStatus;
if (!$indexStatus) {
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocument"), $docId), 'error');
$this->logPendingDocumentInfoStatus($docId, '<output>' . $extractor->output . '</output>', 'error');
}
$extractor->setIndexingStatus($indexStatus);
}
}
$this->executeHook($extractor, 'post_index', $mimeType);
$this->executeHook($extractor, 'post_index');
} else {
$extractor->setExtractionStatus(false);
$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Could not extract contents from document %d"), $docId), 'error');
$this->logPendingDocumentInfoStatus($docId, '<output>' . $extractor->output . '</output>', 'error');
}
$this->executeHook($extractor, 'post_extract', $mimeType);
$this->executeHook($extractor, 'post_extract');
if ($extractor->needsIntermediateSourceFile()) {
@unlink($sourceFile);
}
@unlink($targetFile);
} else {
$indexStatus = $this->indexDiscussion($docId);
$removeFromQueue = $indexStatus;
}
if ($removeFromQueue) {
Indexer::unqueueDocument($docId, sprintf(_kt("Done indexing docid: %d"), $docId));
} else {
if ($this->debug) {
$default->log->debug(sprintf(_kt("Document docid: %d was not removed from the queue as it looks like there was a problem with the extraction process"), $docId));
}
}
}
$tool_content .= action_bar($dynbar);
}
if (isset($_POST['submit'])) {
$subject = trim($_POST['subject']);
$message = trim($_POST['message']);
if (empty($message) or empty($subject)) {
header("Location: viewforum.php?course={$course_code}&forum={$forum_id}&empty=true");
exit;
}
$message = purify($message);
$poster_ip = $_SERVER['REMOTE_ADDR'];
$time = date("Y-m-d H:i:s");
$topic_id = Database::get()->query("INSERT INTO forum_topic (title, poster_id, forum_id, topic_time) VALUES (?s, ?d, ?d, ?t)", $subject, $uid, $forum_id, $time)->lastInsertID;
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_FORUMTOPIC, $topic_id);
$post_id = Database::get()->query("INSERT INTO forum_post (topic_id, post_text, poster_id, post_time, poster_ip) VALUES (?d, ?s, ?d, ?t, ?s)", $topic_id, $message, $uid, $time, $poster_ip)->lastInsertID;
Indexer::queueAsync(Indexer::REQUEST_STORE, Indexer::RESOURCE_FORUMPOST, $post_id);
$forum_user_stats = Database::get()->querySingle("SELECT COUNT(*) as c FROM forum_post \n INNER JOIN forum_topic ON forum_post.topic_id = forum_topic.id\n INNER JOIN forum ON forum.id = forum_topic.forum_id\n WHERE forum_post.poster_id = ?d AND forum.course_id = ?d", $uid, $course_id);
Database::get()->query("DELETE FROM forum_user_stats WHERE user_id = ?d AND course_id = ?d", $uid, $course_id);
Database::get()->query("INSERT INTO forum_user_stats (user_id, num_posts, course_id) VALUES (?d,?d,?d)", $uid, $forum_user_stats->c, $course_id);
Database::get()->query("UPDATE forum_topic\n SET last_post_id = ?d\n WHERE id = ?d\n AND forum_id = ?d", $post_id, $topic_id, $forum_id);
Database::get()->query("UPDATE forum\n SET num_topics = num_topics+1,\n num_posts = num_posts+1,\n last_post_id = ?d\n\t\tWHERE id = ?d", $post_id, $forum_id);
$topic = $topic_id;
$total_forum = get_total_topics($forum_id);
$total_topic = get_total_posts($topic) - 1;
// subtract 1 because we want the number of replies, not the number of posts.
// --------------------------------
// notify users
// --------------------------------
$subject_notify = "{$logo} - {$langNewForumNotify}";
$category_id = forum_category($forum_id);
$cat_name = category_name($category_id);
}
$scheduleIndexing = false;
// indexing was previously off, but now set to on, need to schedule re-indexing
if (!get_config('enable_indexing') && $enable_indexing) {
$scheduleIndexing = true;
Database::get()->query("DELETE FROM idx_queue");
Database::get()->queryFunc("SELECT id FROM course", function($r) {
Database::get()->query("INSERT INTO idx_queue (course_id) VALUES (?d)", $r->id);
});
}
// indexing was previously on, but now set to off, need to empty it
if (get_config('enable_indexing') && !$enable_indexing) {
require_once 'modules/search/indexer.class.php';
Indexer::deleteAll();
}
// update table `config`
foreach ($config_vars as $varname => $what) {
set_config($varname, $GLOBALS[$varname]);
}
// Display result message
Session::flash('scheduleIndexing', $scheduleIndexing);
Session::Messages($langFileUpdatedSuccess, 'alert-success');
redirect_to_home_page('modules/admin/eclassconf.php');
} // end of if($submit)
else {
// Display config.php edit form
