/**
	 * This method will be called before an article is displayed or previewed.
	 * For display and preview we strip out the semantic properties and append them
	 * at the end of the article.
	 *
	 * @param Parser $parser
	 * @param string $text
	 */
	static public function onInternalParseBeforeLinks( &$parser, &$text ) {
		global $smwgStoreAnnotations, $smwgLinksInValues;

		SMWParseData::stripMagicWords( $text, $parser );

		// Store the results if enabled (we have to parse them in any case,
		// in order to clean the wiki source for further processing).
		$smwgStoreAnnotations = smwfIsSemanticsProcessed( $parser->getTitle()->getNamespace() );
		SMWParserExtensions::$mTempStoreAnnotations = true; // used for [[SMW::on]] and [[SMW:off]]

		// Process redirects, if any (it seems that there is indeed no more direct way of getting this info from MW)
		if ( $smwgStoreAnnotations ) {
			$rt = Title::newFromRedirect( $text );
			
			if ( !is_null( $rt ) ) {
				$p = new SMWDIProperty( '_REDI' );
				$di = SMWDIWikiPage::newFromTitle( $rt, '__red' );
				SMWParseData::getSMWData( $parser )->addPropertyObjectValue( $p, $di );
			}
		}

		// only used in subsequent callbacks, forgotten afterwards
		SMWParserExtensions::$mTempParser = $parser;

		// In the regexp matches below, leading ':' escapes the markup, as known for Categories.
		// Parse links to extract semantic properties.
		if ( $smwgLinksInValues ) { // More complex regexp -- lib PCRE may cause segfaults if text is long :-(
			$semanticLinkPattern = '/\[\[                 # Beginning of the link
			                        (?:([^:][^]]*):[=:])+ # Property name (or a list of those)
			                        (                     # After that:
			                          (?:[^|\[\]]         #   either normal text (without |, [ or ])
			                          |\[\[[^]]*\]\]      #   or a [[link]]
			                          |\[[^]]*\]          #   or an [external link]
			                        )*)                   # all this zero or more times
			                        (?:\|([^]]*))?        # Display text (like "text" in [[link|text]]), optional
			                        \]\]                  # End of link
			                        /xu';
			$text = preg_replace_callback( $semanticLinkPattern, array( 'SMWParserExtensions', 'parsePropertiesCallback' ), $text );
		} else { // Simpler regexps -- no segfaults found for those, but no links in values.
			$semanticLinkPattern = '/\[\[                 # Beginning of the link
			                        (?:([^:][^]]*):[=:])+ # Property name (or a list of those)
			                        ([^\[\]]*)            # content: anything but [, |, ]
			                        \]\]                  # End of link
			                        /xu';
			$text = preg_replace_callback( $semanticLinkPattern, array( 'SMWParserExtensions', 'simpleParsePropertiesCallback' ), $text );
		}

		// Add link to RDF to HTML header.
		// TODO: do escaping via Html or Xml class.
		SMWOutputs::requireHeadItem(
			'smw_rdf', '<link rel="alternate" type="application/rdf+xml" title="' .
			htmlspecialchars( $parser->getTitle()->getPrefixedText() ) . '" href="' .
			htmlspecialchars(
				SpecialPage::getTitleFor( 'ExportRDF', $parser->getTitle()->getPrefixedText() )->getLocalUrl( 'xmlmime=rdf' )
			) . "\" />"
		);

		SMWOutputs::commitToParser( $parser );
		return true; // always return true, in order not to stop MW's hook processing!
	}
 /**
  * @since 2.0 made protected; use printAllToFile or printAllToOutput
  */
 protected function printAll($ns_restriction = false, $delay, $delayeach)
 {
     $linkCache = LinkCache::singleton();
     $db = wfGetDB(DB_SLAVE);
     $this->delay_flush = 10;
     $this->serializer->startSerialization();
     $this->serializer->serializeExpData(SMWExporter::getInstance()->getOntologyExpData(''));
     $end = $db->selectField('page', 'max(page_id)', false, __METHOD__);
     $a_count = 0;
     // DEBUG
     $d_count = 0;
     // DEBUG
     $delaycount = $delayeach;
     for ($id = 1; $id <= $end; $id += 1) {
         $title = Title::newFromID($id);
         if (is_null($title) || !smwfIsSemanticsProcessed($title->getNamespace())) {
             continue;
         }
         if (!self::fitsNsRestriction($ns_restriction, $title->getNamespace())) {
             continue;
         }
         $a_count += 1;
         // DEBUG
         $diPage = SMWDIWikiPage::newFromTitle($title);
         $this->queuePage($diPage, 1);
         while (count($this->element_queue) > 0) {
             $diPage = reset($this->element_queue);
             $this->serializePage($diPage, $diPage->recdepth);
             // resolve dependencies that will otherwise not be printed
             foreach ($this->element_queue as $key => $diaux) {
                 if (!smwfIsSemanticsProcessed($diaux->getNamespace()) || !self::fitsNsRestriction($ns_restriction, $diaux->getNamespace())) {
                     // Note: we do not need to check the cache to guess if an element was already
                     // printed. If so, it would not be included in the queue in the first place.
                     $d_count += 1;
                     // DEBUG
                 } else {
                     // don't carry values that you do not want to export (yet)
                     unset($this->element_queue[$key]);
                 }
             }
             // sleep each $delaycount for $delay µs to be nice to the server
             if ($delaycount-- < 0 && $delayeach != 0) {
                 usleep($delay);
                 $delaycount = $delayeach;
             }
         }
         $this->flush();
         $linkCache->clear();
     }
     $this->serializer->finishSerialization();
     $this->flush(true);
 }
 /**
  * @see smwfIsSemanticsProcessed
  *
  * FIXME Delete this test in 1.11
  *
  * @since 1.9
  */
 public function testSmwfIsSemanticsProcessed()
 {
     $result = smwfIsSemanticsProcessed(NS_MAIN);
     $this->assertInternalType('boolean', $result);
     $this->assertTrue($result);
 }
Exemple #4
0
 /**
  * This function takes care of storing the collected semantic data and takes
  * care of clearing out any outdated entries for the processed page. It assume that
  * parsing has happened and that all relevant data is contained in the provided parser
  * output.
  *
  * Optionally, this function also takes care of triggering indirect updates that might be
  * needed for overall database consistency. If the saved page describes a property or data type,
  * the method checks whether the property type, the data type, the allowed values, or the
  * conversion factors have changed. If so, it triggers SMWUpdateJobs for the relevant articles,
  * which then asynchronously update the semantic data in the database.
  *
  * @param $parseroutput ParserOutput object that contains the results of parsing which will
  * be stored.
  * @param $title Title object specifying the page that should be saved.
  * @param $makejobs Bool stating whether jobs should be created to trigger further updates if
  * this appears to be necessary after this update.
  *
  * @todo FIXME: Some job generations here might create too many jobs at once on a large wiki. Use incremental jobs instead.
  */
 public static function storeData($parseroutput, Title $title, $makejobs = true)
 {
     global $smwgEnableUpdateJobs, $smwgDeclarationProperties, $smwgPageSpecialProperties;
     $semdata = $parseroutput->mSMWData;
     $namespace = $title->getNamespace();
     $processSemantics = smwfIsSemanticsProcessed($namespace);
     if (!isset($semdata)) {
         // no data at all?
         $semdata = new SMWSemanticData(SMWDIWikiPage::newFromTitle($title));
     }
     if ($processSemantics) {
         $props = array();
         foreach ($smwgPageSpecialProperties as $propId) {
             // Do not calculate the same property again.
             if (array_key_exists($propId, $props)) {
                 continue;
             }
             // Remember the property is processed.
             $props[$propId] = true;
             $prop = new SMWDIProperty($propId);
             if (count($semdata->getPropertyValues($prop)) > 0) {
                 continue;
             }
             // Calculate property value.
             $value = null;
             switch ($propId) {
                 case '_MDAT':
                     $timestamp = Revision::getTimeStampFromID($title, $title->getLatestRevID());
                     $value = self::getDataItemFromMWTimestamp($timestamp);
                     break;
                 case '_CDAT':
                     $timestamp = $title->getFirstRevision()->getTimestamp();
                     $value = self::getDataItemFromMWTimestamp($timestamp);
                     break;
                 case '_NEWP':
                     $value = new SMWDIBoolean($title->isNewPage());
                     break;
                 case '_LEDT':
                     $revision = Revision::newFromId($title->getLatestRevID());
                     $user = User::newFromId($revision->getUser());
                     $value = SMWDIWikiPage::newFromTitle($user->getUserPage());
                     break;
             }
             if (!is_null($value)) {
                 $semdata->addPropertyObjectValue($prop, $value);
             }
             // Issue error or warning?
         }
         // foreach
     } else {
         // data found, but do all operations as if it was empty
         $semdata = new SMWSemanticData($semdata->getSubject());
     }
     // Check if the semantic data has been changed.
     // Sets the updateflag to true if so.
     // Careful: storage access must happen *before* the storage update;
     // even finding uses of a property fails after its type was changed.
     $updatejobflag = false;
     $jobs = array();
     if ($makejobs && $smwgEnableUpdateJobs && $namespace == SMW_NS_PROPERTY) {
         // If it is a property, then we need to check if the type or the allowed values have been changed.
         $ptype = new SMWDIProperty('_TYPE');
         $oldtype = smwfGetStore()->getPropertyValues($semdata->getSubject(), $ptype);
         $newtype = $semdata->getPropertyValues($ptype);
         if (!self::equalDatavalues($oldtype, $newtype)) {
             $updatejobflag = true;
         } else {
             foreach ($smwgDeclarationProperties as $prop) {
                 $pv = new SMWDIProperty($prop);
                 $oldvalues = smwfGetStore()->getPropertyValues($semdata->getSubject(), $pv);
                 $newvalues = $semdata->getPropertyValues($pv);
                 $updatejobflag = !self::equalDatavalues($oldvalues, $newvalues);
             }
         }
         if ($updatejobflag) {
             $prop = new SMWDIProperty($title->getDBkey());
             $subjects = smwfGetStore()->getAllPropertySubjects($prop);
             foreach ($subjects as $subject) {
                 $subjectTitle = $subject->getTitle();
                 if (!is_null($subjectTitle)) {
                     // wikia change start - jobqueue migration
                     $task = new \Wikia\Tasks\Tasks\JobWrapperTask();
                     $task->call('SMWUpdateJob', $subjectTitle);
                     $jobs[] = $task;
                     // wikia change end
                 }
             }
             wfRunHooks('smwUpdatePropertySubjects', array(&$jobs));
             $subjects = smwfGetStore()->getPropertySubjects(new SMWDIProperty('_ERRP'), $semdata->getSubject());
             foreach ($subjects as $subject) {
                 $subjectTitle = $subject->getTitle();
                 if (!is_null($subjectTitle)) {
                     // wikia change start - jobqueue migration
                     $task = new \Wikia\Tasks\Tasks\JobWrapperTask();
                     $task->call('SMWUpdateJob', $subjectTitle);
                     $jobs[] = $task;
                     // wikia change end
                 }
             }
         }
     } elseif ($makejobs && $smwgEnableUpdateJobs && $namespace == SMW_NS_TYPE) {
         // if it is a type we need to check if the conversion factors have been changed
         $pconv = new SMWDIProperty('_CONV');
         $ptype = new SMWDIProperty('_TYPE');
         $oldfactors = smwfGetStore()->getPropertyValues($semdata->getSubject(), $pconv);
         $newfactors = $semdata->getPropertyValues($pconv);
         $updatejobflag = !self::equalDatavalues($oldfactors, $newfactors);
         if ($updatejobflag) {
             $store = smwfGetStore();
             /// FIXME: this will kill large wikis! Use incremental updates!
             $dv = SMWDataValueFactory::newTypeIdValue('__typ', $title->getDBkey());
             $proppages = $store->getPropertySubjects($ptype, $dv);
             foreach ($proppages as $proppage) {
                 $propertyTitle = $proppage->getTitle();
                 if (!is_null($propertyTitle)) {
                     // wikia change start - jobqueue migration
                     $task = new \Wikia\Tasks\Tasks\JobWrapperTask();
                     $task->call('SMWUpdateJob', $propertyTitle);
                     $jobs[] = $task;
                     // wikia change end
                 }
                 $prop = new SMWDIProperty($proppage->getDBkey());
                 $subjects = $store->getAllPropertySubjects($prop);
                 foreach ($subjects as $subject) {
                     $subjectTitle = $subject->getTitle();
                     if (!is_null($subjectTitle)) {
                         // wikia change start - jobqueue migration
                         $task = new \Wikia\Tasks\Tasks\JobWrapperTask();
                         $task->call('SMWUpdateJob', $subjectTitle);
                         $jobs[] = $task;
                         // wikia change end
                     }
                 }
                 $subjects = smwfGetStore()->getPropertySubjects(new SMWDIProperty('_ERRP'), $prop->getWikiPageValue());
                 foreach ($subjects as $subject) {
                     $subjectTitle = $subject->getTitle();
                     if (!is_null($subjectTitle)) {
                         // wikia change start - jobqueue migration
                         $task = new \Wikia\Tasks\Tasks\JobWrapperTask();
                         $task->call('SMWUpdateJob', $subjectTitle);
                         $jobs[] = $task;
                         // wikia change end
                     }
                 }
             }
         }
     }
     // Actually store semantic data, or at least clear it if needed
     if ($processSemantics) {
         smwfGetStore()->updateData($semdata);
     } else {
         smwfGetStore()->clearData($semdata->getSubject());
     }
     // Finally trigger relevant Updatejobs if necessary
     if ($updatejobflag) {
         // wikia change start - jobqueue migration
         \Wikia\Tasks\Tasks\BaseTask::batch($jobs);
         // wikia change end
     }
     return true;
 }
         $doit = true;
     }
     if ($properties && $ns == SMW_NS_PROPERTY) {
         $doit = true;
     }
     if ($types && $ns == SMW_NS_TYPE) {
         $doit = true;
     }
     if (!$doit) {
         continue;
     }
 }
 if ($verbose) {
     print "({$num_files}) Processing page with ID " . $id . " ...\n";
 }
 if (smwfIsSemanticsProcessed($title->getNamespace())) {
     $revision = Revision::newFromTitle($title);
     if ($revision === NULL) {
         continue;
     }
     $wgParser->parse($revision->getText(), $title, $options, true, true, $revision->getID());
     SMWFactbox::storeData($title, true);
     // sleep to be nice to the server
     if ($delay !== false && ($num_files + 1) % 100 === 0) {
         usleep($delay);
     }
 } else {
     smwfGetStore()->deleteSubject($title);
     // sleep to be nice to the server
     // (for this case, sleep only every 1000 pages, so that large chunks of e.g. messages are processed more quickly)
     if ($delay !== false && ($num_files + 1) % 1000 === 0) {