/**
* @since 2.4
*
* @return QuerySegmentListProcessor
*/
public function newQuerySegmentListProcessor()
{
$connection = $this->store->getConnection('mw.db.queryengine');
$temporaryTableBuilder = $this->newTemporaryTableBuilder();
$hierarchyTempTableBuilder = new HierarchyTempTableBuilder($connection, $temporaryTableBuilder);
$hierarchyTempTableBuilder->setPropertyHierarchyTableDefinition($this->store->findPropertyTableID(new DIProperty('_SUBP')), $this->applicationFactory->getSettings()->get('smwgQSubpropertyDepth'));
$hierarchyTempTableBuilder->setClassHierarchyTableDefinition($this->store->findPropertyTableID(new DIProperty('_SUBC')), $this->applicationFactory->getSettings()->get('smwgQSubcategoryDepth'));
$querySegmentListProcessor = new QuerySegmentListProcessor($connection, $temporaryTableBuilder, $hierarchyTempTableBuilder);
return $querySegmentListProcessor;
}
/**
* Create an array of rows to insert into property tables in order to
* store the given SMWSemanticData. The given $sid (subject page id) is
* used directly and must belong to the subject of the data container.
* Sortkeys are ignored since they are not stored in a property table
* but in the ID table.
*
* The returned array uses property table names as keys and arrays of
* table rows as values. Each table row is an array mapping column
* names to values.
*
* @note Property tables that do not use ids as subjects are ignored.
* This just excludes redirects that are handled differently anyway;
* it would not make a difference to include them here.
*
* @since 1.8
* @param integer $sid
* @param SMWSemanticData $data
* @return array
*/
protected function preparePropertyTableInserts($sid, SMWSemanticData $data)
{
$updates = array();
$subject = $data->getSubject();
$propertyTables = $this->store->getPropertyTables();
foreach ($data->getProperties() as $property) {
$tableId = $this->store->findPropertyTableID($property);
if (is_null($tableId)) {
// not stored in a property table, e.g., sortkeys
continue;
}
$propertyTable = $propertyTables[$tableId];
if (!$propertyTable->usesIdSubject()) {
// not using subject ids, e.g., redirects
continue;
}
$insertValues = array('s_id' => $sid);
if (!$propertyTable->isFixedPropertyTable()) {
$insertValues['p_id'] = $this->store->smwIds->makeSMWPropertyID($property);
}
foreach ($data->getPropertyValues($property) as $di) {
if ($di instanceof SMWDIError) {
// ignore error values
continue;
}
if (!array_key_exists($propertyTable->getName(), $updates)) {
$updates[$propertyTable->getName()] = array();
}
$diHandler = $this->store->getDataItemHandlerForDIType($di->getDIType());
// Note that array_merge creates a new array; not overwriting past entries here
$insertValues = array_merge($insertValues, $diHandler->getInsertValues($di));
$updates[$propertyTable->getName()][] = $insertValues;
}
}
// Special handling of Concepts
if ($subject->getNamespace() === SMW_NS_CONCEPT && $subject->getSubobjectName() == '') {
$this->prepareConceptTableInserts($sid, $updates);
}
return $updates;
}
public function execute()
{
$options = array();
if ($this->hasArg(0)) {
if ($this->hasArg(1)) {
$options['LIMIT'] = $this->getArg(0);
$options['OFFSET'] = $this->getArg(1);
}
}
$dbw = wfGetDB(DB_MASTER);
$res = $dbw->select('smw_ids', array('smw_id', 'smw_title', 'smw_sortkey'), array('smw_namespace' => SMW_NS_PROPERTY), __METHOD__, $options);
$proptables = SMWSQLStore3::getPropertyTables();
foreach ($res as $row) {
$di = new SMWDIProperty($row->smw_title);
$tableId = SMWSQLStore3::findPropertyTableID($di);
$proptable = $proptables[$tableId];
$propRow = $dbw->selectRow($proptable->name, 'Count(*) as count', $proptable->fixedproperty ? array() : array('p_id' => $row->smw_id), __METHOD__);
echo 'Usage count for ' . $row->smw_title . ' is ' . $propRow->count . "\n";
$dbw->replace('smw_stats', 'pid', array('pid' => $row->smw_id, 'usage_count' => $propRow->count), __METHOD__);
}
$dbw->freeResult($res);
}
/**
* Helper function to compute from and where strings for a DB query so that
* only rows of the given value object match. The parameter $tableindex
* counts that tables used in the query to avoid duplicate table names. The
* parameter $proptable provides the SMWSQLStore3Table object that is
* queried.
*
* @todo Maybe do something about redirects. The old code was
* $oid = $this->store->smwIds->getSMWPageID($value->getDBkey(),$value->getNamespace(),$value->getInterwiki(),false);
*
* @note This method cannot handle DIContainer objects with sortkey
* properties correctly. This should never occur, but it would be good
* to fail in a more controlled way if it ever does.
*
* @param string $from
* @param string $where
* @param TableDefinition $propTable
* @param SMWDataItem $value
* @param integer $tableIndex
*/
private function prepareValueQuery(&$from, &$where, TableDefinition $propTable, $value, $tableIndex = 1)
{
$db = $this->store->getConnection();
if ($value instanceof SMWDIContainer) {
// recursive handling of containers
$keys = array_keys($propTable->getFields($this->store));
$joinfield = "t{$tableIndex}." . reset($keys);
// this must be a type 'p' object
$proptables = $this->store->getPropertyTables();
$semanticData = $value->getSemanticData();
foreach ($semanticData->getProperties() as $subproperty) {
$tableid = $this->store->findPropertyTableID($subproperty);
$subproptable = $proptables[$tableid];
foreach ($semanticData->getPropertyValues($subproperty) as $subvalue) {
$tableIndex++;
if ($subproptable->usesIdSubject()) {
// simply add property table to check values
$from .= " INNER JOIN " . $db->tableName($subproptable->getName()) . " AS t{$tableIndex} ON t{$tableIndex}.s_id={$joinfield}";
} else {
// exotic case with table that uses subject title+namespace in container object (should never happen in SMW core)
$from .= " INNER JOIN " . $db->tableName(SMWSql3SmwIds::TABLE_NAME) . " AS ids{$tableIndex} ON ids{$tableIndex}.smw_id={$joinfield}" . " INNER JOIN " . $db->tableName($subproptable->getName()) . " AS t{$tableIndex} ON " . "t{$tableIndex}.s_title=ids{$tableIndex}.smw_title AND t{$tableIndex}.s_namespace=ids{$tableIndex}.smw_namespace";
}
if (!$subproptable->isFixedPropertyTable()) {
// the ID we get should be !=0, so no point in filtering the converse
$where .= ($where ? ' AND ' : '') . "t{$tableIndex}.p_id=" . $db->addQuotes($this->store->smwIds->getSMWPropertyID($subproperty));
}
$this->prepareValueQuery($from, $where, $subproptable, $subvalue, $tableIndex);
}
}
} elseif (!is_null($value)) {
// add conditions for given value
$diHandler = $this->store->getDataItemHandlerForDIType($value->getDIType());
foreach ($diHandler->getWhereConds($value) as $fieldname => $value) {
$where .= ($where ? ' AND ' : '') . "t{$tableIndex}.{$fieldname}=" . $db->addQuotes($value);
}
}
}
public function execute()
{
global $wgDBtype;
if ($this->hasOption('setup')) {
$store = new SMWSQLStore3();
// Lets do a drop to ensure the user doesn't has any Store3 tables already (happens when running this script twice)
$tables = array('smw_stats');
foreach (SMWSQLStore3::getPropertyTables() as $proptable) {
$tables[] = $proptable->name;
}
$dbw = wfGetDB(DB_MASTER);
foreach ($tables as $table) {
$name = $dbw->tableName($table);
$dbw->query('DROP TABLE ' . ($wgDBtype == 'postgres' ? '' : 'IF EXISTS ') . $name, 'SMWMigrate::drop');
}
$store->setup();
//enter user defined properties into smw_stats (internal ones are handled by setup already )
$query = 'Replace into ' . $dbw->tableName('smw_stats') . ' (pid,usage_count) Select smw_id,0 from ' . $dbw->tableName('smw_ids') . ' where smw_namespace = ' . SMW_NS_PROPERTY . ' and smw_iw = "" ';
$dbw->query($query, 'SMWMigrate:commandLine');
} elseif ($this->hasOption('migrate')) {
$options = array();
if ($this->hasArg(0)) {
if ($this->hasArg(1)) {
$options['LIMIT'] = $this->getArg(0);
$options['OFFSET'] = $this->getArg(1);
}
}
$dbw = wfGetDB(DB_MASTER);
$oldStore = new SMWSQLStore2();
$newStore = new SMWSQLStore3();
$proptables = SMWSQLStore3::getPropertyTables();
//get properties
$res = $dbw->select('smw_ids', array('smw_id', 'smw_title', 'smw_namespace'), array('smw_namespace' => SMW_NS_PROPERTY), __METHOD__, $options);
foreach ($res as $row) {
$property = new SMWDIProperty($row->smw_title);
echo 'Now migrating data for Property ' . $property->getLabel() . " into Store3 \n";
//get the table
$tableId = SMWSQLStore3::findPropertyTableID($property);
$proptable = $proptables[$tableId];
//get the DIHandler
$dataItemId = SMWDataValueFactory::getDataItemId($property->findPropertyTypeId());
$diHandler = $newStore->getDataItemHandlerForDIType($dataItemId);
$subjects = $oldStore->getPropertySubjects($property, null);
$insertions = array();
foreach ($subjects as $subject) {
$sid = $newStore->makeSMWPageID($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $subject->getSubobjectName(), true, str_replace('_', ' ', $subject->getDBkey()) . $subject->getSubobjectName());
//now prepare udpates
$propvals = $oldStore->getPropertyValues($subject, $property);
$uvals = $proptable->idsubject ? array('s_id' => $sid) : array('s_title' => $subject->getDBkey(), 's_namespace' => $subject->getNamespace());
if ($proptable->fixedproperty == false) {
$uvals['p_id'] = $newStore->makeSMWPropertyID($property);
}
foreach ($propvals as $propval) {
$uvals = array_merge($uvals, $diHandler->getInsertValues($propval));
$insertions[] = $uvals;
}
}
// now write to the DB for all subjects (is this too much?)
$dbw->insert($proptable->name, $insertions, "SMW::migrate{$proptable->name}");
}
$dbw->freeResult($res);
} else {
echo "Sorry I refuse to work without any options currently";
}
}
/**
* Modify the given query object to account for some property condition for
* the given property. If it is not possible to generate a query for the
* given data, the query type is changed to SMWSQLStore3Query::Q_NOQUERY. Callers need
* to check for this and discard the query in this case.
*
* @note This method does not support sortkey (_SKEY) property queries,
* since they do not have a normal property table. This should not be a
* problem since comparators on sortkeys are supported indirectly when
* using comparators on wikipages. There is no reason to create any
* query with _SKEY ad users cannot do so either (no user label).
*
* @since 1.8
*/
protected function compileSomePropertyDescription(SMWSQLStore3Query $query, SMWSomeProperty $description)
{
$property = $description->getProperty();
$tableid = SMWSQLStore3::findPropertyTableID($property);
if ($tableid === '') {
// Give up
$query->type = SMWSQLStore3Query::Q_NOQUERY;
return;
}
$proptables = SMWSQLStore3::getPropertyTables();
$proptable = $proptables[$tableid];
if (!$proptable->usesIdSubject()) {
// no queries with such tables
// (only redirects are affected in practice)
$query->type = SMWSQLStore3Query::Q_NOQUERY;
return;
}
$typeid = $property->findPropertyTypeID();
$diType = SMWDataValueFactory::getDataItemId($typeid);
if ($property->isInverse() && $diType != SMWDataItem::TYPE_WIKIPAGE) {
// can only invert properties that point to pages
$query->type = SMWSQLStore3Query::Q_NOQUERY;
return;
}
$diHandler = $this->m_store->getDataItemHandlerForDIType($diType);
$indexField = $diHandler->getIndexField();
$sortkey = $property->getKey();
// TODO: strictly speaking, the DB key is not what we want here, since sortkey is based on a "wiki value"
// *** Now construct the query ... ***//
$query->jointable = $proptable->getName();
// *** Add conditions for selecting rows for this property ***//
if (!$proptable->isFixedPropertyTable()) {
$pid = $this->m_store->smwIds->getSMWPropertyID($property);
// Construct property hierarchy:
$pqid = SMWSQLStore3Query::$qnum;
$pquery = new SMWSQLStore3Query();
$pquery->type = SMWSQLStore3Query::Q_PROP_HIERARCHY;
$pquery->joinfield = array($pid);
$query->components[$pqid] = "{$query->alias}.p_id";
$this->m_queries[$pqid] = $pquery;
// Alternative code without property hierarchies:
// $query->where = "{$query->alias}.p_id=" . $this->m_dbs->addQuotes( $pid );
}
// else: no property column, no hierarchy queries
// *** Add conditions on the value of the property ***//
if ($diType == SMWDataItem::TYPE_WIKIPAGE) {
$o_id = $indexField;
if ($property->isInverse()) {
$s_id = $o_id;
$o_id = 's_id';
} else {
$s_id = 's_id';
}
$query->joinfield = "{$query->alias}.{$s_id}";
// process page description like main query
$sub = $this->compileQueries($description->getDescription());
if ($sub >= 0) {
$query->components[$sub] = "{$query->alias}.{$o_id}";
}
if (array_key_exists($sortkey, $this->m_sortkeys)) {
// TODO: This SMW IDs table is possibly duplicated in the query.
// Example: [[has capital::!Berlin]] with sort=has capital
// Can we prevent that? (PERFORMANCE)
$query->from = ' INNER JOIN ' . $this->m_dbs->tableName(SMWSql3SmwIds::tableName) . " AS ids{$query->alias} ON ids{$query->alias}.smw_id={$query->alias}.{$o_id}";
$query->sortfields[$sortkey] = "ids{$query->alias}.smw_sortkey";
}
} else {
// non-page value description
$query->joinfield = "{$query->alias}.s_id";
$this->compilePropertyValueDescription($query, $description->getDescription(), $proptable, $diHandler, 'AND');
if (array_key_exists($sortkey, $this->m_sortkeys)) {
$query->sortfields[$sortkey] = "{$query->alias}.{$indexField}";
}
}
}
public function getStatistics()
{
wfProfileIn('SMWSQLStore3::getStatistics (SMW)');
$dbr = wfGetDB(DB_SLAVE);
$result = array();
$proptables = SMWSQLStore3::getPropertyTables();
// count number of declared properties by counting "has type" annotations
$typeprop = new SMWDIProperty('_TYPE');
$typetable = $proptables[SMWSQLStore3::findPropertyTableID($typeprop)];
$res = $dbr->select($typetable->name, 'COUNT(s_id) AS count', array(), 'SMW::getStatistics');
$row = $dbr->fetchObject($res);
$result['DECLPROPS'] = $row->count;
$dbr->freeResult($res);
// count property uses by counting rows in property tables,
// count used properties by counting distinct properties in each table
$result['PROPUSES'] = 0;
$result['USEDPROPS'] = 0;
foreach (SMWSQLStore3::getPropertyTables() as $proptable) {
/// Note: subproperties that are part of container values are counted individually;
/// It does not seem to be important to filter them by adding more conditions.
$res = $dbr->select($proptable->name, 'COUNT(*) AS count', '', 'SMW::getStatistics');
$row = $dbr->fetchObject($res);
$result['PROPUSES'] += $row->count;
$dbr->freeResult($res);
if ($proptable->fixedproperty == false) {
$res = $dbr->select($proptable->name, 'COUNT(DISTINCT(p_id)) AS count', '', 'SMW::getStatistics');
$row = $dbr->fetchObject($res);
$result['USEDPROPS'] += $row->count;
} else {
$res = $dbr->select($proptable->name, '*', '', 'SMW::getStatistics', array('LIMIT' => 1));
if ($dbr->numRows($res) > 0) {
$result['USEDPROPS']++;
}
}
$dbr->freeResult($res);
}
wfProfileOut('SMWSQLStore3::getStatistics (SMW)');
return $result;
}
/**
* Compute statistics for all the properties, basically update the count in smw_stats
*/
protected function computeStats($verbose, $db)
{
$this->reportProgress("Computing property statistics.\n", $verbose);
$res = $db->select('smw_ids', array('smw_id', 'smw_title', 'smw_sortkey'), array('smw_namespace' => SMW_NS_PROPERTY), __METHOD__);
$proptables = SMWSQLStore3::getPropertyTables();
$count = 0;
foreach ($res as $row) {
$count++;
try {
$di = new SMWDIProperty($row->smw_title);
} catch (SMWDataItemException $e) {
$this->reportProgress("Warning: Could not create a property for key\"{$row->smw_title}\" ({$e->getMessage()}) \n", $verbose);
continue;
}
$tableId = SMWSQLStore3::findPropertyTableID($di);
$proptable = $proptables[$tableId];
$propRow = $db->selectRow($proptable->name, 'Count(*) as count', $proptable->fixedproperty ? array() : array('p_id' => $row->smw_id), __METHOD__);
$db->replace('smw_stats', 'pid', array('pid' => $row->smw_id, 'usage_count' => $propRow->count), __METHOD__);
}
$db->freeResult($res);
$this->reportProgress("Updated statistics for {$count} Properties.\n", $verbose);
}
/**
* Extend the given update array to account for the data in the
* SMWSemanticData object. The subject page of the data container is
* ignored, and the given $sid (subject page id) is used directly. If
* this ID is 0, then $subject is used to find an ID. This is usually
* the case for all internal objects (subobjects) that are created in
* writing sub-SemanticData.
*
* The function returns the id that was used for writing. Especially,
* any newly created internal id is returned.
*
* @param $updates array
* @param $data SMWSemanticData
* @param $sid integer pre-computed id if available or 0 if ID should be sought
* @param $subject SMWDIWikiPage subject to which the data refers
*/
protected function prepareDBUpdates(&$updates, SMWSemanticData $data, $sid, SMWDIWikiPage $subject)
{
if ($sid == 0) {
$sid = $this->store->smwIds->makeSMWPageID($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $subject->getSubobjectName(), true, str_replace('_', ' ', $subject->getDBkey()) . $subject->getSubobjectName());
}
$proptables = SMWSQLStore3::getPropertyTables();
foreach ($data->getProperties() as $property) {
if ($property->getKey() == '_SKEY' || $property->getKey() == '_REDI') {
continue;
// skip these here, we store them differently
}
$tableid = SMWSQLStore3::findPropertyTableID($property);
$proptable = $proptables[$tableid];
///TODO check needed if subject is null (would happen if a user defined proptable with !idsubject was used on an internal object -- currently this is not possible
$uvals = $proptable->idsubject ? array('s_id' => $sid) : array('s_title' => $subject->getDBkey(), 's_namespace' => $subject->getNamespace());
if ($proptable->fixedproperty == false) {
$uvals['p_id'] = $this->store->smwIds->makeSMWPropertyID($property);
}
foreach ($data->getPropertyValues($property) as $di) {
if ($di instanceof SMWDIError) {
// error values, ignore
continue;
}
$diHandler = $this->store->getDataItemHandlerForDIType($di->getDIType());
$uvals = array_merge($uvals, $diHandler->getInsertValues($di));
if (!array_key_exists($proptable->name, $updates)) {
$updates[$proptable->name] = array();
}
$updates[$proptable->name][] = $uvals;
}
}
// Special handling of Concepts
if ($subject->getNamespace() == SMW_NS_CONCEPT && $subject->getSubobjectName() == '') {
if (array_key_exists('smw_fpt_conc', $updates) && count($updates['smw_fpt_conc']) != 0) {
$updates['smw_fpt_conc'] = end($updates['smw_fpt_conc']);
unset($updates['smw_fpt_conc']['cache_date']);
unset($updates['smw_fpt_conc']['cache_count']);
} else {
$updates['smw_fpt_conc'] = array('concept_txt' => '', 'concept_docu' => '', 'concept_features' => 0, 'concept_size' => -1, 'concept_depth' => -1);
}
}
return $sid;
}
