/** * Adds a test document to the index and to the in-memory store. * * @param int $index Document index * @param stdClass $course Course object * @param string $content Text content * @param int $cmid If specified, cmid of document */ private static function add_test_document($index, $course, $content, $cmid = 0) { $doc = new local_ousearch_document(); $doc->courseid = $course->id; $doc->plugin = 'test_yeartablestest'; $doc->timemodified = $index; $doc->set_int_refs($index); if ($cmid) { $doc->coursemoduleid = $cmid; } $doc->update('Title', $content); self::$testdocuments[$index] = array('title' => 'Title', 'content' => $content); }
/** * Saves a new version of the given named page within a subwiki. Can create * a new page or just add a new version to an existing one. In case of * failure, ends up calling error() rather than returning something. * * @param object $course Course object * @param object $cm Course-module object * @param object $ouwiki OU wiki object * @param object $subwiki Subwiki object * @param string $pagename Name of page (NO SLASHES) * @param string $content XHTML Content (NO SLASHES) * @param int $changestart For section changes. Start position of change. (-1 if not section change) * @param int $changesize Size of changed section. * @param int $changeprevsize Previous size of changed section * @param bool $nouser If true, creates as system * @param object $formdata if coming from edit will have content embedded media and attachments * @param int revertversionid if coming from revert.php will have an older versionid */ function ouwiki_save_new_version($course, $cm, $ouwiki, $subwiki, $pagename, $content, $changestart = -1, $changesize = -1, $changeprevsize = -1, $nouser = null, $formdata = null, $revertversionid = null) { global $DB, $USER; global $ouwikiinternalre, $ouwiki_count; // Nasty but I can't think of a better way! $transaction = $DB->start_delegated_transaction(); // Find page if it exists $pageversion = ouwiki_get_current_page($subwiki, $pagename, OUWIKI_GETPAGE_CREATE); $previousversionid = null; if ($pageversion->currentversionid) { $previousversionid = $pageversion->currentversionid; } // Analyse content for HTML headings that don't already have an ID. // These are all assigned unique, fairly short IDs. // Get number of version [guarantees in-page uniqueness of generated IDs] $versionnumber = $DB->count_records('ouwiki_versions', array('pageid' => $pageversion->pageid)); // Remove any spaces from annotation tags that were added for editing or by users // and remove any duplicate annotation tags $pattern = '~<span\\b.id=\\"annotation(.+?)\\">.*?</span>~'; $replace = '<span id="annotation$1"></span>'; $content = preg_replace($pattern, $replace, $content); unset($pattern, $replace, $used); // Get rid of any heading tags that only contain whitespace $emptypatterns = array(); for ($i = 1; $i <= 6; $i++) { $emptypatterns[] = '~<h' . $i . '[^>]*>\\s*(<br[^>]*>\\s*)*</h' . $i . '>~'; } $content = preg_replace($emptypatterns, '', $content); // List all headings that already have IDs, to check for duplicates $matches = array(); preg_match_all('|<h[1-9] id="ouw_s(.*?)">(.*?)</h[1-9]>|', $content, $matches, PREG_SET_ORDER | PREG_OFFSET_CAPTURE); // Organise list by ID $byid = array(); foreach ($matches as $index => $data) { $id = $data[1][0]; if (!array_key_exists($id, $byid)) { $byid[$id] = array(); } $byid[$id][] = $index; } // Handle any duplicates $deletebits = array(); foreach ($byid as $id => $duplicates) { if (count($duplicates) > 1) { // We have a duplicate. By default, keep the first one $keep = $duplicates[0]; // See if there is a title entry in the database for it $knowntitle = $DB->get_field('ouwiki_sections', 'title', array('xhtmlid' => $id, 'pageid' => $pageversion->pageid)); if ($knowntitle) { foreach ($duplicates as $duplicate) { $title = ouwiki_get_section_title(null, null, $matches[$duplicate][2][0]); if ($title === $knowntitle) { $keep = $duplicate; break; } } } foreach ($duplicates as $duplicate) { if ($duplicate !== $keep) { $deletebits[] = (object) array('startbyte' => $matches[$duplicate][1][1] - 10, 'bytes' => strlen($matches[$duplicate][1][0]) + 11); } } } } // Were there any? if (count($deletebits) > 0) { // Sort in reverse order of starting position usort($deletebits, 'ouwiki_internal_sort_deletions'); // Delete each bit foreach ($deletebits as $deletebit) { $content = substr($content, 0, $deletebit->startbyte) . substr($content, $deletebit->startbyte + $deletebit->bytes); } } // Replace existing empty headings with an ID including version count plus another index $ouwiki_count = 0; $ouwikiinternalre->version = $versionnumber; $ouwikiinternalre->count = 0; $sizebefore = strlen($content); $content = preg_replace_callback('/<h([1-9])>/', 'ouwiki_internal_re_headings', $content); $sizeafter = strlen($content); // Replace wiki links to [[Start page]] with the correct (non // language-specific) format [[]] $regex = str_replace('.*?', preg_quote(get_string('startpage', 'ouwiki')), OUWIKI_LINKS_SQUAREBRACKETS) . 'ui'; $newcontent = @preg_replace($regex, '[[]]', $content); if ($newcontent === null) { // Unicode support not available! Change the regex and try again $regex = preg_replace('~ui$~', 'i', $regex); $newcontent = preg_replace($regex, '[[]]', $content); } $content = $newcontent; // Create version $version = new StdClass(); $version->pageid = $pageversion->pageid; $version->xhtml = $content; // May be altered later (see below) $version->xhtmlformat = FORMAT_MOODLE; // Using fixed value here is a bit rubbish $version->timecreated = time(); $version->wordcount = ouwiki_count_words($content); $version->previousversionid = $previousversionid; if (!$nouser) { $version->userid = $USER->id; } if ($changestart != -1) { $version->changestart = $changestart; // In tracking the new size, account for any added headings etc $version->changesize = $changesize + ($sizeafter - $sizebefore); $version->changeprevsize = $changeprevsize; } try { $versionid = $DB->insert_record('ouwiki_versions', $version); // if firstversionid is already set in the current page use that // else this is a new page and version entirely if (!$pageversion->firstversionid) { $DB->set_field('ouwiki_pages', 'firstversionid', $versionid, array('id' => $version->pageid)); } } catch (Exception $e) { ouwiki_dberror($e); } // information needed for attachments $fs = get_file_storage(); $modcontext = get_context_instance(CONTEXT_MODULE, $cm->id); $prevversion = $revertversionid ? $revertversionid : $pageversion->versionid; // save new files connected with the version from the formdata if set if ($formdata) { $formdata->content = file_save_draft_area_files($formdata->content['itemid'], $modcontext->id, 'mod_ouwiki', 'content', $versionid, array('subdirs' => 0), $content); if ($content !== $formdata->content) { $DB->set_field('ouwiki_versions', 'xhtml', $formdata->content, array('id' => $versionid)); } if (isset($formdata->attachments)) { file_save_draft_area_files($formdata->attachments, $modcontext->id, 'mod_ouwiki', 'attachment', $versionid, array('subdirs' => 0)); } } else { // need to copy over attached files from the previous version when // editing without using form if ($oldfiles = $fs->get_area_files($modcontext->id, 'mod_ouwiki', 'attachment', $prevversion)) { foreach ($oldfiles as $oldfile) { // copy this file to the version record. $fs->create_file_from_storedfile(array('contextid' => $modcontext->id, 'filearea' => 'attachment', 'itemid' => $versionid), $oldfile); } } if ($oldfiles = $fs->get_area_files($modcontext->id, 'mod_ouwiki', 'content', $prevversion)) { foreach ($oldfiles as $oldfile) { // copy this file to the version record. $fs->create_file_from_storedfile(array('contextid' => $modcontext->id, 'filearea' => 'content', 'itemid' => $versionid), $oldfile); } } } // Update latest version $DB->set_field('ouwiki_pages', 'currentversionid', $versionid, array('id' => $pageversion->pageid)); // Analyse for links $wikilinks = array(); $externallinks = array(); // Wiki links: ordinary [[links]] $matches = array(); preg_match_all(OUWIKI_LINKS_SQUAREBRACKETS, $content, $matches, PREG_PATTERN_ORDER); foreach ($matches[1] as $match) { // Convert to page name (this also removes HTML tags etc) $wikilinks[] = ouwiki_get_wiki_link_details($match)->page; } // Note that we used to support CamelCase links but have removed support because: // 1. Confusing: students type JavaScript or MySpace and don't expect it to become a link // 2. Not accessible: screenreaders cannot cope with run-together words, and // dyslexic students can have difficulty reading them // External links preg_match_all('/<a [^>]*href=(?:(?:\'(.*?)\')|(?:"(.*?))")/', $content, $matches, PREG_PATTERN_ORDER); foreach ($matches[1] as $match) { if ($match) { $externallinks[] = html_entity_decode($match); } } foreach ($matches[2] as $match) { if ($match) { $externallinks[] = html_entity_decode($match); } } // Add link records $link = new StdClass(); $link->fromversionid = $versionid; foreach ($wikilinks as $targetpage) { if (!empty($targetpage)) { $pagerecord = $DB->get_record_select('ouwiki_pages', 'subwikiid = ? AND UPPER(title) = UPPER(?)', array($subwiki->id, $targetpage)); if ($pagerecord) { $pageid = $pagerecord->id; } else { $pageid = false; } } else { $pageid = $DB->get_field_select('ouwiki_pages', 'id', 'subwikiid = ? AND title IS NULL', array($subwiki->id)); } if ($pageid) { $link->topageid = $pageid; $link->tomissingpage = null; } else { $link->topageid = null; $link->tomissingpage = strtoupper($targetpage); } try { $link->id = $DB->insert_record('ouwiki_links', $link); } catch (Exception $e) { ouwiki_dberror($e); } } $link->topageid = null; $link->tomissingpage = null; $tl = textlib_get_instance(); foreach ($externallinks as $url) { // Restrict length of URL if ($tl->strlen($url) > 255) { $url = $tl->substr($url, 0, 255); } $link->tourl = $url; try { $link->id = $DB->insert_record('ouwiki_links', $link); } catch (Exception $e) { ouwiki_dberror($e); } } // Inform search, if installed if (ouwiki_search_installed()) { $doc = new local_ousearch_document(); $doc->init_module_instance('ouwiki', $cm); if ($subwiki->groupid) { $doc->set_group_id($subwiki->groupid); } $doc->set_string_ref($pageversion->title === '' ? null : $pageversion->title); if ($subwiki->userid) { $doc->set_user_id($subwiki->userid); } $title = $pageversion->title; $doc->update($title, $content); } $transaction->allow_commit(); }
/** * Update all wiki documents for ousearch. * * @param bool $feedback If true, prints feedback as HTML list items * @param int $courseid If specified, restricts to particular courseid */ function ouwiki_ousearch_update_all($feedback = null, $courseid = 0) { global $CFG, $DB; require_once $CFG->dirroot . '/mod/ouwiki/locallib.php'; // Get list of all wikis. We need the coursemodule data plus // the type of subwikis $coursecriteria = $courseid === 0 ? '' : 'AND cm.course = ' . $courseid; $sql = "SELECT cm.id, cm.course, cm.instance, w.subwikis\n FROM {modules} m\n INNER JOIN {course_modules} cm ON cm.module = m.id\n INNER JOIN {ouwiki} w ON cm.instance = w.id\n WHERE m.name = 'ouwiki' {$coursecriteria}"; $coursemodules = $DB->get_records_sql($sql, array()); if (!$coursemodules) { return; } if ($feedback) { print '<li><strong>' . count($coursemodules) . '</strong> wikis to process.</li>'; $dotcount = 0; } $count = 0; foreach ($coursemodules as $coursemodule) { // This condition is needed because if somebody creates some stuff // then changes the wiki type, it actually keeps the old bits // in the database. Maybe it shouldn't, not sure. switch ($coursemodule->subwikis) { case OUWIKI_SUBWIKIS_SINGLE: $where = "sw.userid IS NULL AND sw.groupid IS NULL"; break; case OUWIKI_SUBWIKIS_GROUPS: $where = "sw.userid IS NULL AND sw.groupid IS NOT NULL"; break; case OUWIKI_SUBWIKIS_INDIVIDUAL: $where = "sw.userid IS NOT NULL AND sw.groupid IS NULL"; break; } // Get all pages in that wiki $sql = "SELECT p.id, p.title, v.xhtml, v.timecreated, sw.groupid, sw.userid\n FROM {ouwiki_subwikis} sw\n INNER JOIN {ouwiki_pages} p ON p.subwikiid = sw.id\n INNER JOIN {ouwiki_versions} v ON v.id = p.currentversionid\n WHERE sw.wikiid = ? AND {$where}"; $rs = $DB->get_recordset_sql($sql, array($coursemodule->instance)); foreach ($rs as $result) { // Update the page for search $doc = new local_ousearch_document(); $doc->init_module_instance('ouwiki', $coursemodule); if ($result->groupid) { $doc->set_group_id($result->groupid); } if ($result->title) { $doc->set_string_ref($result->title); } if ($result->userid) { $doc->set_user_id($result->userid); } $title = $result->title ? $result->title : ''; $doc->update($title, $result->xhtml, $result->timecreated); } $rs->close(); $count++; if ($feedback) { if ($dotcount == 0) { print '<li>'; } print '.'; $dotcount++; if ($dotcount == 20 || $count == count($coursemodules)) { print 'done ' . $count . '</li>'; $dotcount = 0; } flush(); } } }
public function test_ousearch_query() { global $DB; $this->resetAfterTest(); // Create a bunch of search documents within test_zombie plugin. self::$zombiedocuments = array(1 => (object) array('title' => 'Document title', 'content' => 'First zombie document'), 2 => (object) array('title' => 'Another title', 'content' => 'Title title first'), 3 => (object) array('title' => 'Document title', 'content' => 'Not a zombie document title'), 4 => (object) array('title' => 'Delete me', 'content' => 'Delete me'), 100 => (object) array('title' => 'Bottle quantity', 'content' => 'There are this many bottles on the wall: 0')); for ($i = 101; $i <= 199; $i++) { self::$zombiedocuments[$i] = self::$zombiedocuments[100]; self::$zombiedocuments[$i]->content = str_replace(': 0', ': ' . ($i - 100), self::$zombiedocuments[$i]->content); } foreach (self::$zombiedocuments as $key => $content) { $doc = new local_ousearch_document(); $doc->init_test('zombie'); $doc->set_int_refs($key); $doc->update($content->title, $content->content, null, null, null); } // Search for single unique term. $result = $this->do_zombie_query('not'); $this->assertTrue($result->success); $this->assertEquals(array(3), $this->get_result_ids($result)); // Search for nonexistent word. $result = $this->do_zombie_query('xyzzy'); $this->assertFalse($result->success); $this->assertEquals('xyzzy', $result->problemword); // Search for nothing. $result = $this->do_zombie_query(' '); $this->assertFalse($result->success); $this->assertEquals('', $result->problemword); // Search for pair of terms. $result = $this->do_zombie_query('first document'); $this->assertTrue($result->success); $this->assertEquals(array(1), $this->get_result_ids($result)); // Search for quoted terms. $result = $this->do_zombie_query('"title title"'); $this->assertTrue($result->success); $this->assertEquals(array(2), $this->get_result_ids($result)); // Negative terms. $result = $this->do_zombie_query('title -not'); $this->assertTrue($result->success); $this->assertEquals(array(1, 2), $this->get_result_ids($result)); // Negative quoted terms. $result = $this->do_zombie_query('title -"not frog"'); $this->assertTrue($result->success); $this->assertEquals(array(1, 2, 3), $this->get_result_ids($result)); $result = $this->do_zombie_query('title -"not a"'); $this->assertTrue($result->success); $this->assertEquals(array(1, 2), $this->get_result_ids($result)); // Deleting stale results (those which the module responsible can no // longer find). $before = $DB->count_records('local_ousearch_documents'); unset(self::$zombiedocuments[4]); $result = $this->do_zombie_query('delete'); $this->assertEquals(1, count(phpunit_util::get_debugging_messages())); phpunit_util::reset_debugging(); $this->assertTrue($result->success); $this->assertEquals(array(), $this->get_result_ids($result)); $this->assertEquals($before - 1, $DB->count_records('local_ousearch_documents')); // Ranking based on title vs content and number of occurrences. $result = $this->do_zombie_query('title'); $this->assertTrue($result->success); $this->assertEquals(array(1, 2, 3), $this->get_result_ids($result)); $this->assertEquals(2, $result->results[0]->intref1); $this->assertEquals(18, $result->results[0]->totalscore); // Managing result lists. $found = array(); $dbstart = 0; for ($i = 0; $i < 10; $i++) { $result = $this->do_zombie_query('bottles', $dbstart); $this->assertTrue($result->success); $this->assertEquals(10, count($result->results)); foreach ($result->results as $thing) { $found[$thing->intref1] = true; } $dbstart = $result->dbstart; } $this->assertEquals(100, count($found)); $result = $this->do_zombie_query('bottles', $dbstart); $this->assertTrue($result->success); $this->assertEquals(0, count($result->results)); }