/** * @since 2.4 * * @return QuerySegmentListProcessor */ public function newQuerySegmentListProcessor() { $connection = $this->store->getConnection('mw.db.queryengine'); $temporaryTableBuilder = $this->newTemporaryTableBuilder(); $hierarchyTempTableBuilder = new HierarchyTempTableBuilder($connection, $temporaryTableBuilder); $hierarchyTempTableBuilder->setPropertyHierarchyTableDefinition($this->store->findPropertyTableID(new DIProperty('_SUBP')), $this->applicationFactory->getSettings()->get('smwgQSubpropertyDepth')); $hierarchyTempTableBuilder->setClassHierarchyTableDefinition($this->store->findPropertyTableID(new DIProperty('_SUBC')), $this->applicationFactory->getSettings()->get('smwgQSubcategoryDepth')); $querySegmentListProcessor = new QuerySegmentListProcessor($connection, $temporaryTableBuilder, $hierarchyTempTableBuilder); return $querySegmentListProcessor; }
/** * Create an array of rows to insert into property tables in order to * store the given SMWSemanticData. The given $sid (subject page id) is * used directly and must belong to the subject of the data container. * Sortkeys are ignored since they are not stored in a property table * but in the ID table. * * The returned array uses property table names as keys and arrays of * table rows as values. Each table row is an array mapping column * names to values. * * @note Property tables that do not use ids as subjects are ignored. * This just excludes redirects that are handled differently anyway; * it would not make a difference to include them here. * * @since 1.8 * @param integer $sid * @param SMWSemanticData $data * @return array */ protected function preparePropertyTableInserts($sid, SMWSemanticData $data) { $updates = array(); $subject = $data->getSubject(); $propertyTables = $this->store->getPropertyTables(); foreach ($data->getProperties() as $property) { $tableId = $this->store->findPropertyTableID($property); if (is_null($tableId)) { // not stored in a property table, e.g., sortkeys continue; } $propertyTable = $propertyTables[$tableId]; if (!$propertyTable->usesIdSubject()) { // not using subject ids, e.g., redirects continue; } $insertValues = array('s_id' => $sid); if (!$propertyTable->isFixedPropertyTable()) { $insertValues['p_id'] = $this->store->smwIds->makeSMWPropertyID($property); } foreach ($data->getPropertyValues($property) as $di) { if ($di instanceof SMWDIError) { // ignore error values continue; } if (!array_key_exists($propertyTable->getName(), $updates)) { $updates[$propertyTable->getName()] = array(); } $diHandler = $this->store->getDataItemHandlerForDIType($di->getDIType()); // Note that array_merge creates a new array; not overwriting past entries here $insertValues = array_merge($insertValues, $diHandler->getInsertValues($di)); $updates[$propertyTable->getName()][] = $insertValues; } } // Special handling of Concepts if ($subject->getNamespace() === SMW_NS_CONCEPT && $subject->getSubobjectName() == '') { $this->prepareConceptTableInserts($sid, $updates); } return $updates; }
public function execute() { $options = array(); if ($this->hasArg(0)) { if ($this->hasArg(1)) { $options['LIMIT'] = $this->getArg(0); $options['OFFSET'] = $this->getArg(1); } } $dbw = wfGetDB(DB_MASTER); $res = $dbw->select('smw_ids', array('smw_id', 'smw_title', 'smw_sortkey'), array('smw_namespace' => SMW_NS_PROPERTY), __METHOD__, $options); $proptables = SMWSQLStore3::getPropertyTables(); foreach ($res as $row) { $di = new SMWDIProperty($row->smw_title); $tableId = SMWSQLStore3::findPropertyTableID($di); $proptable = $proptables[$tableId]; $propRow = $dbw->selectRow($proptable->name, 'Count(*) as count', $proptable->fixedproperty ? array() : array('p_id' => $row->smw_id), __METHOD__); echo 'Usage count for ' . $row->smw_title . ' is ' . $propRow->count . "\n"; $dbw->replace('smw_stats', 'pid', array('pid' => $row->smw_id, 'usage_count' => $propRow->count), __METHOD__); } $dbw->freeResult($res); }
/** * Helper function to compute from and where strings for a DB query so that * only rows of the given value object match. The parameter $tableindex * counts that tables used in the query to avoid duplicate table names. The * parameter $proptable provides the SMWSQLStore3Table object that is * queried. * * @todo Maybe do something about redirects. The old code was * $oid = $this->store->smwIds->getSMWPageID($value->getDBkey(),$value->getNamespace(),$value->getInterwiki(),false); * * @note This method cannot handle DIContainer objects with sortkey * properties correctly. This should never occur, but it would be good * to fail in a more controlled way if it ever does. * * @param string $from * @param string $where * @param TableDefinition $propTable * @param SMWDataItem $value * @param integer $tableIndex */ private function prepareValueQuery(&$from, &$where, TableDefinition $propTable, $value, $tableIndex = 1) { $db = $this->store->getConnection(); if ($value instanceof SMWDIContainer) { // recursive handling of containers $keys = array_keys($propTable->getFields($this->store)); $joinfield = "t{$tableIndex}." . reset($keys); // this must be a type 'p' object $proptables = $this->store->getPropertyTables(); $semanticData = $value->getSemanticData(); foreach ($semanticData->getProperties() as $subproperty) { $tableid = $this->store->findPropertyTableID($subproperty); $subproptable = $proptables[$tableid]; foreach ($semanticData->getPropertyValues($subproperty) as $subvalue) { $tableIndex++; if ($subproptable->usesIdSubject()) { // simply add property table to check values $from .= " INNER JOIN " . $db->tableName($subproptable->getName()) . " AS t{$tableIndex} ON t{$tableIndex}.s_id={$joinfield}"; } else { // exotic case with table that uses subject title+namespace in container object (should never happen in SMW core) $from .= " INNER JOIN " . $db->tableName(SMWSql3SmwIds::TABLE_NAME) . " AS ids{$tableIndex} ON ids{$tableIndex}.smw_id={$joinfield}" . " INNER JOIN " . $db->tableName($subproptable->getName()) . " AS t{$tableIndex} ON " . "t{$tableIndex}.s_title=ids{$tableIndex}.smw_title AND t{$tableIndex}.s_namespace=ids{$tableIndex}.smw_namespace"; } if (!$subproptable->isFixedPropertyTable()) { // the ID we get should be !=0, so no point in filtering the converse $where .= ($where ? ' AND ' : '') . "t{$tableIndex}.p_id=" . $db->addQuotes($this->store->smwIds->getSMWPropertyID($subproperty)); } $this->prepareValueQuery($from, $where, $subproptable, $subvalue, $tableIndex); } } } elseif (!is_null($value)) { // add conditions for given value $diHandler = $this->store->getDataItemHandlerForDIType($value->getDIType()); foreach ($diHandler->getWhereConds($value) as $fieldname => $value) { $where .= ($where ? ' AND ' : '') . "t{$tableIndex}.{$fieldname}=" . $db->addQuotes($value); } } }
public function execute() { global $wgDBtype; if ($this->hasOption('setup')) { $store = new SMWSQLStore3(); // Lets do a drop to ensure the user doesn't has any Store3 tables already (happens when running this script twice) $tables = array('smw_stats'); foreach (SMWSQLStore3::getPropertyTables() as $proptable) { $tables[] = $proptable->name; } $dbw = wfGetDB(DB_MASTER); foreach ($tables as $table) { $name = $dbw->tableName($table); $dbw->query('DROP TABLE ' . ($wgDBtype == 'postgres' ? '' : 'IF EXISTS ') . $name, 'SMWMigrate::drop'); } $store->setup(); //enter user defined properties into smw_stats (internal ones are handled by setup already ) $query = 'Replace into ' . $dbw->tableName('smw_stats') . ' (pid,usage_count) Select smw_id,0 from ' . $dbw->tableName('smw_ids') . ' where smw_namespace = ' . SMW_NS_PROPERTY . ' and smw_iw = "" '; $dbw->query($query, 'SMWMigrate:commandLine'); } elseif ($this->hasOption('migrate')) { $options = array(); if ($this->hasArg(0)) { if ($this->hasArg(1)) { $options['LIMIT'] = $this->getArg(0); $options['OFFSET'] = $this->getArg(1); } } $dbw = wfGetDB(DB_MASTER); $oldStore = new SMWSQLStore2(); $newStore = new SMWSQLStore3(); $proptables = SMWSQLStore3::getPropertyTables(); //get properties $res = $dbw->select('smw_ids', array('smw_id', 'smw_title', 'smw_namespace'), array('smw_namespace' => SMW_NS_PROPERTY), __METHOD__, $options); foreach ($res as $row) { $property = new SMWDIProperty($row->smw_title); echo 'Now migrating data for Property ' . $property->getLabel() . " into Store3 \n"; //get the table $tableId = SMWSQLStore3::findPropertyTableID($property); $proptable = $proptables[$tableId]; //get the DIHandler $dataItemId = SMWDataValueFactory::getDataItemId($property->findPropertyTypeId()); $diHandler = $newStore->getDataItemHandlerForDIType($dataItemId); $subjects = $oldStore->getPropertySubjects($property, null); $insertions = array(); foreach ($subjects as $subject) { $sid = $newStore->makeSMWPageID($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $subject->getSubobjectName(), true, str_replace('_', ' ', $subject->getDBkey()) . $subject->getSubobjectName()); //now prepare udpates $propvals = $oldStore->getPropertyValues($subject, $property); $uvals = $proptable->idsubject ? array('s_id' => $sid) : array('s_title' => $subject->getDBkey(), 's_namespace' => $subject->getNamespace()); if ($proptable->fixedproperty == false) { $uvals['p_id'] = $newStore->makeSMWPropertyID($property); } foreach ($propvals as $propval) { $uvals = array_merge($uvals, $diHandler->getInsertValues($propval)); $insertions[] = $uvals; } } // now write to the DB for all subjects (is this too much?) $dbw->insert($proptable->name, $insertions, "SMW::migrate{$proptable->name}"); } $dbw->freeResult($res); } else { echo "Sorry I refuse to work without any options currently"; } }
/** * Modify the given query object to account for some property condition for * the given property. If it is not possible to generate a query for the * given data, the query type is changed to SMWSQLStore3Query::Q_NOQUERY. Callers need * to check for this and discard the query in this case. * * @note This method does not support sortkey (_SKEY) property queries, * since they do not have a normal property table. This should not be a * problem since comparators on sortkeys are supported indirectly when * using comparators on wikipages. There is no reason to create any * query with _SKEY ad users cannot do so either (no user label). * * @since 1.8 */ protected function compileSomePropertyDescription(SMWSQLStore3Query $query, SMWSomeProperty $description) { $property = $description->getProperty(); $tableid = SMWSQLStore3::findPropertyTableID($property); if ($tableid === '') { // Give up $query->type = SMWSQLStore3Query::Q_NOQUERY; return; } $proptables = SMWSQLStore3::getPropertyTables(); $proptable = $proptables[$tableid]; if (!$proptable->usesIdSubject()) { // no queries with such tables // (only redirects are affected in practice) $query->type = SMWSQLStore3Query::Q_NOQUERY; return; } $typeid = $property->findPropertyTypeID(); $diType = SMWDataValueFactory::getDataItemId($typeid); if ($property->isInverse() && $diType != SMWDataItem::TYPE_WIKIPAGE) { // can only invert properties that point to pages $query->type = SMWSQLStore3Query::Q_NOQUERY; return; } $diHandler = $this->m_store->getDataItemHandlerForDIType($diType); $indexField = $diHandler->getIndexField(); $sortkey = $property->getKey(); // TODO: strictly speaking, the DB key is not what we want here, since sortkey is based on a "wiki value" // *** Now construct the query ... ***// $query->jointable = $proptable->getName(); // *** Add conditions for selecting rows for this property ***// if (!$proptable->isFixedPropertyTable()) { $pid = $this->m_store->smwIds->getSMWPropertyID($property); // Construct property hierarchy: $pqid = SMWSQLStore3Query::$qnum; $pquery = new SMWSQLStore3Query(); $pquery->type = SMWSQLStore3Query::Q_PROP_HIERARCHY; $pquery->joinfield = array($pid); $query->components[$pqid] = "{$query->alias}.p_id"; $this->m_queries[$pqid] = $pquery; // Alternative code without property hierarchies: // $query->where = "{$query->alias}.p_id=" . $this->m_dbs->addQuotes( $pid ); } // else: no property column, no hierarchy queries // *** Add conditions on the value of the property ***// if ($diType == SMWDataItem::TYPE_WIKIPAGE) { $o_id = $indexField; if ($property->isInverse()) { $s_id = $o_id; $o_id = 's_id'; } else { $s_id = 's_id'; } $query->joinfield = "{$query->alias}.{$s_id}"; // process page description like main query $sub = $this->compileQueries($description->getDescription()); if ($sub >= 0) { $query->components[$sub] = "{$query->alias}.{$o_id}"; } if (array_key_exists($sortkey, $this->m_sortkeys)) { // TODO: This SMW IDs table is possibly duplicated in the query. // Example: [[has capital::!Berlin]] with sort=has capital // Can we prevent that? (PERFORMANCE) $query->from = ' INNER JOIN ' . $this->m_dbs->tableName(SMWSql3SmwIds::tableName) . " AS ids{$query->alias} ON ids{$query->alias}.smw_id={$query->alias}.{$o_id}"; $query->sortfields[$sortkey] = "ids{$query->alias}.smw_sortkey"; } } else { // non-page value description $query->joinfield = "{$query->alias}.s_id"; $this->compilePropertyValueDescription($query, $description->getDescription(), $proptable, $diHandler, 'AND'); if (array_key_exists($sortkey, $this->m_sortkeys)) { $query->sortfields[$sortkey] = "{$query->alias}.{$indexField}"; } } }
public function getStatistics() { wfProfileIn('SMWSQLStore3::getStatistics (SMW)'); $dbr = wfGetDB(DB_SLAVE); $result = array(); $proptables = SMWSQLStore3::getPropertyTables(); // count number of declared properties by counting "has type" annotations $typeprop = new SMWDIProperty('_TYPE'); $typetable = $proptables[SMWSQLStore3::findPropertyTableID($typeprop)]; $res = $dbr->select($typetable->name, 'COUNT(s_id) AS count', array(), 'SMW::getStatistics'); $row = $dbr->fetchObject($res); $result['DECLPROPS'] = $row->count; $dbr->freeResult($res); // count property uses by counting rows in property tables, // count used properties by counting distinct properties in each table $result['PROPUSES'] = 0; $result['USEDPROPS'] = 0; foreach (SMWSQLStore3::getPropertyTables() as $proptable) { /// Note: subproperties that are part of container values are counted individually; /// It does not seem to be important to filter them by adding more conditions. $res = $dbr->select($proptable->name, 'COUNT(*) AS count', '', 'SMW::getStatistics'); $row = $dbr->fetchObject($res); $result['PROPUSES'] += $row->count; $dbr->freeResult($res); if ($proptable->fixedproperty == false) { $res = $dbr->select($proptable->name, 'COUNT(DISTINCT(p_id)) AS count', '', 'SMW::getStatistics'); $row = $dbr->fetchObject($res); $result['USEDPROPS'] += $row->count; } else { $res = $dbr->select($proptable->name, '*', '', 'SMW::getStatistics', array('LIMIT' => 1)); if ($dbr->numRows($res) > 0) { $result['USEDPROPS']++; } } $dbr->freeResult($res); } wfProfileOut('SMWSQLStore3::getStatistics (SMW)'); return $result; }
/** * Compute statistics for all the properties, basically update the count in smw_stats */ protected function computeStats($verbose, $db) { $this->reportProgress("Computing property statistics.\n", $verbose); $res = $db->select('smw_ids', array('smw_id', 'smw_title', 'smw_sortkey'), array('smw_namespace' => SMW_NS_PROPERTY), __METHOD__); $proptables = SMWSQLStore3::getPropertyTables(); $count = 0; foreach ($res as $row) { $count++; try { $di = new SMWDIProperty($row->smw_title); } catch (SMWDataItemException $e) { $this->reportProgress("Warning: Could not create a property for key\"{$row->smw_title}\" ({$e->getMessage()}) \n", $verbose); continue; } $tableId = SMWSQLStore3::findPropertyTableID($di); $proptable = $proptables[$tableId]; $propRow = $db->selectRow($proptable->name, 'Count(*) as count', $proptable->fixedproperty ? array() : array('p_id' => $row->smw_id), __METHOD__); $db->replace('smw_stats', 'pid', array('pid' => $row->smw_id, 'usage_count' => $propRow->count), __METHOD__); } $db->freeResult($res); $this->reportProgress("Updated statistics for {$count} Properties.\n", $verbose); }
/** * Extend the given update array to account for the data in the * SMWSemanticData object. The subject page of the data container is * ignored, and the given $sid (subject page id) is used directly. If * this ID is 0, then $subject is used to find an ID. This is usually * the case for all internal objects (subobjects) that are created in * writing sub-SemanticData. * * The function returns the id that was used for writing. Especially, * any newly created internal id is returned. * * @param $updates array * @param $data SMWSemanticData * @param $sid integer pre-computed id if available or 0 if ID should be sought * @param $subject SMWDIWikiPage subject to which the data refers */ protected function prepareDBUpdates(&$updates, SMWSemanticData $data, $sid, SMWDIWikiPage $subject) { if ($sid == 0) { $sid = $this->store->smwIds->makeSMWPageID($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $subject->getSubobjectName(), true, str_replace('_', ' ', $subject->getDBkey()) . $subject->getSubobjectName()); } $proptables = SMWSQLStore3::getPropertyTables(); foreach ($data->getProperties() as $property) { if ($property->getKey() == '_SKEY' || $property->getKey() == '_REDI') { continue; // skip these here, we store them differently } $tableid = SMWSQLStore3::findPropertyTableID($property); $proptable = $proptables[$tableid]; ///TODO check needed if subject is null (would happen if a user defined proptable with !idsubject was used on an internal object -- currently this is not possible $uvals = $proptable->idsubject ? array('s_id' => $sid) : array('s_title' => $subject->getDBkey(), 's_namespace' => $subject->getNamespace()); if ($proptable->fixedproperty == false) { $uvals['p_id'] = $this->store->smwIds->makeSMWPropertyID($property); } foreach ($data->getPropertyValues($property) as $di) { if ($di instanceof SMWDIError) { // error values, ignore continue; } $diHandler = $this->store->getDataItemHandlerForDIType($di->getDIType()); $uvals = array_merge($uvals, $diHandler->getInsertValues($di)); if (!array_key_exists($proptable->name, $updates)) { $updates[$proptable->name] = array(); } $updates[$proptable->name][] = $uvals; } } // Special handling of Concepts if ($subject->getNamespace() == SMW_NS_CONCEPT && $subject->getSubobjectName() == '') { if (array_key_exists('smw_fpt_conc', $updates) && count($updates['smw_fpt_conc']) != 0) { $updates['smw_fpt_conc'] = end($updates['smw_fpt_conc']); unset($updates['smw_fpt_conc']['cache_date']); unset($updates['smw_fpt_conc']['cache_count']); } else { $updates['smw_fpt_conc'] = array('concept_txt' => '', 'concept_docu' => '', 'concept_features' => 0, 'concept_size' => -1, 'concept_depth' => -1); } } return $sid; }