/** * Sets up the property tables. * * @since 1.8 * @param array $dbtypes * @param DatabaseBase|Database $db * @param SMWSQLStore3SetupHandlers|null $reportTo */ protected function setupPropertyTables(array $dbtypes, $db, SMWSQLStore3SetupHandlers $reportTo = null) { $addedCustomTypeSignatures = false; foreach (SMWSQLStore3::getPropertyTables() as $proptable) { $diHandler = $this->store->getDataItemHandlerForDIType($proptable->getDiType()); // Prepare indexes. By default, property-value tables // have the following indexes: // // sp: getPropertyValues(), getSemanticData(), getProperties() // po: ask, getPropertySubjects() // // The "p" component is omitted for tables with fixed property. $indexes = array(); if ($proptable->usesIdSubject()) { $fieldarray = array('s_id' => $dbtypes['p'] . ' NOT NULL'); $indexes['sp'] = 's_id'; } else { $fieldarray = array('s_title' => $dbtypes['t'] . ' NOT NULL', 's_namespace' => $dbtypes['n'] . ' NOT NULL'); $indexes['sp'] = 's_title,s_namespace'; } $indexes['po'] = $diHandler->getIndexField(); if (!$proptable->isFixedPropertyTable()) { $fieldarray['p_id'] = $dbtypes['p'] . ' NOT NULL'; $indexes['po'] = 'p_id,' . $indexes['po']; $indexes['sp'] = $indexes['sp'] . ',p_id'; } // TODO Special handling; concepts should be handled differently // in the future. See comments in SMW_DIHandler_Concept.php. if ($proptable->getDiType() == SMWDataItem::TYPE_CONCEPT) { unset($indexes['po']); } $indexes = array_merge($indexes, $diHandler->getTableIndexes()); $indexes = array_unique($indexes); foreach ($diHandler->getTableFields() as $fieldname => $typeid) { // If the type signature is not recognized and the custom signatures have not been added, add them. if (!$addedCustomTypeSignatures && !array_key_exists($typeid, $dbtypes)) { wfRunHooks('SMWCustomSQLStoreFieldType', array(&$dbtypes)); $addedCustomTypeSignatures = true; } // Only add the type when the signature was recognized, otherwise ignore it silently. if (array_key_exists($typeid, $dbtypes)) { $fieldarray[$fieldname] = $dbtypes[$typeid]; } } SMWSQLHelpers::setupTable($proptable->getName(), $fieldarray, $db, $reportTo); SMWSQLHelpers::setupIndex($proptable->getName(), $indexes, $db, $reportTo); } }
/** * Get the array of all stored values for some property. * * @since 1.8 * * @param DIProperty $property * * @return array of SMWDataItem */ public function getPropertyValues(DIProperty $property) { if ($property->isInverse()) { // we never have any data for inverses return array(); } if (array_key_exists($property->getKey(), $this->mStubPropVals)) { // Not catching exception here; the $this->unstubProperty($property->getKey(), $property); $propertyTypeId = $property->findPropertyTypeID(); $propertyDiId = DataTypeRegistry::getInstance()->getDataItemId($propertyTypeId); foreach ($this->mStubPropVals[$property->getKey()] as $dbkeys) { try { $diHandler = $this->store->getDataItemHandlerForDIType($propertyDiId); $di = $diHandler->dataItemFromDBKeys($dbkeys); if ($this->mNoDuplicates) { $this->mPropVals[$property->getKey()][$di->getHash()] = $di; } else { $this->mPropVals[$property->getKey()][] = $di; } } catch (SMWDataItemException $e) { // ignore data } } unset($this->mStubPropVals[$property->getKey()]); } return parent::getPropertyValues($property); }
/** * Method to get all subobjects for a given subject. * * @since 1.8 * @param SMWDIWikiPage $subject * * @return array of smw_id => SMWDIWikiPage */ protected function getSubobjects(SMWDIWikiPage $subject) { $db = $this->store->getConnection(); $res = $db->select($db->tablename(SMWSql3SmwIds::tableName), 'smw_id,smw_subobject,smw_sortkey', 'smw_title = ' . $db->addQuotes($subject->getDBkey()) . ' AND ' . 'smw_namespace = ' . $db->addQuotes($subject->getNamespace()) . ' AND ' . 'smw_iw = ' . $db->addQuotes($subject->getInterwiki()) . ' AND ' . 'smw_subobject != ' . $db->addQuotes(''), __METHOD__); $diHandler = $this->store->getDataItemHandlerForDIType(SMWDataItem::TYPE_WIKIPAGE); $subobjects = array(); foreach ($res as $row) { $subobjects[$row->smw_id] = $diHandler->dataItemFromDBKeys(array($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $row->smw_sortkey, $row->smw_subobject)); } $db->freeResult($res); return $subobjects; }
/** * Extend the given update array to account for the data in the * SMWSemanticData object. The subject page of the data container is * ignored, and the given $sid (subject page id) is used directly. If * this ID is 0, then $subject is used to find an ID. This is usually * the case for all internal objects (subobjects) that are created in * writing sub-SemanticData. * * The function returns the id that was used for writing. Especially, * any newly created internal id is returned. * * @param $updates array * @param $data SMWSemanticData * @param $sid integer pre-computed id if available or 0 if ID should be sought * @param $subject SMWDIWikiPage subject to which the data refers */ protected function prepareDBUpdates(&$updates, SMWSemanticData $data, $sid, SMWDIWikiPage $subject) { if ($sid == 0) { $sid = $this->store->smwIds->makeSMWPageID($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $subject->getSubobjectName(), true, str_replace('_', ' ', $subject->getDBkey()) . $subject->getSubobjectName()); } $proptables = SMWSQLStore3::getPropertyTables(); foreach ($data->getProperties() as $property) { if ($property->getKey() == '_SKEY' || $property->getKey() == '_REDI') { continue; // skip these here, we store them differently } $tableid = SMWSQLStore3::findPropertyTableID($property); $proptable = $proptables[$tableid]; ///TODO check needed if subject is null (would happen if a user defined proptable with !idsubject was used on an internal object -- currently this is not possible $uvals = $proptable->idsubject ? array('s_id' => $sid) : array('s_title' => $subject->getDBkey(), 's_namespace' => $subject->getNamespace()); if ($proptable->fixedproperty == false) { $uvals['p_id'] = $this->store->smwIds->makeSMWPropertyID($property); } foreach ($data->getPropertyValues($property) as $di) { if ($di instanceof SMWDIError) { // error values, ignore continue; } $diHandler = $this->store->getDataItemHandlerForDIType($di->getDIType()); $uvals = array_merge($uvals, $diHandler->getInsertValues($di)); if (!array_key_exists($proptable->name, $updates)) { $updates[$proptable->name] = array(); } $updates[$proptable->name][] = $uvals; } } // Special handling of Concepts if ($subject->getNamespace() == SMW_NS_CONCEPT && $subject->getSubobjectName() == '') { if (array_key_exists('smw_fpt_conc', $updates) && count($updates['smw_fpt_conc']) != 0) { $updates['smw_fpt_conc'] = end($updates['smw_fpt_conc']); unset($updates['smw_fpt_conc']['cache_date']); unset($updates['smw_fpt_conc']['cache_count']); } else { $updates['smw_fpt_conc'] = array('concept_txt' => '', 'concept_docu' => '', 'concept_features' => 0, 'concept_size' => -1, 'concept_depth' => -1); } } return $sid; }
/** * Create an array of rows to insert into property tables in order to * store the given SMWSemanticData. The given $sid (subject page id) is * used directly and must belong to the subject of the data container. * Sortkeys are ignored since they are not stored in a property table * but in the ID table. * * The returned array uses property table names as keys and arrays of * table rows as values. Each table row is an array mapping column * names to values. * * @note Property tables that do not use ids as subjects are ignored. * This just excludes redirects that are handled differently anyway; * it would not make a difference to include them here. * * @since 1.8 * @param integer $sid * @param SMWSemanticData $data * @param DatabaseBase $dbr used for reading only * @return array */ protected function preparePropertyTableInserts($sid, SMWSemanticData $data, DatabaseBase $dbr) { $updates = array(); $subject = $data->getSubject(); $propertyTables = SMWSQLStore3::getPropertyTables(); foreach ($data->getProperties() as $property) { $tableId = SMWSQLStore3::findPropertyTableID($property); if (is_null($tableId)) { // not stored in a property table, e.g., sortkeys continue; } $propertyTable = $propertyTables[$tableId]; if (!$propertyTable->usesIdSubject()) { // not using subject ids, e.g., redirects continue; } $insertValues = array('s_id' => $sid); if (!$propertyTable->isFixedPropertyTable()) { $insertValues['p_id'] = $this->store->smwIds->makeSMWPropertyID($property); } foreach ($data->getPropertyValues($property) as $di) { if ($di instanceof SMWDIError) { // ignore error values continue; } if (!array_key_exists($propertyTable->getName(), $updates)) { $updates[$propertyTable->getName()] = array(); } $diHandler = $this->store->getDataItemHandlerForDIType($di->getDIType()); // Note that array_merge creates a new array; not overwriting past entries here $insertValues = array_merge($insertValues, $diHandler->getInsertValues($di)); $insertValueKey = self::makeDatabaseRowKey($insertValues); $updates[$propertyTable->getName()][$insertValueKey] = $insertValues; } } // Special handling of Concepts if ($subject->getNamespace() == SMW_NS_CONCEPT && $subject->getSubobjectName() == '') { $this->prepareConceptTableInserts($sid, $updates, $dbr); } return $updates; }
/** * Helper function to compute from and where strings for a DB query so that * only rows of the given value object match. The parameter $tableindex * counts that tables used in the query to avoid duplicate table names. The * parameter $proptable provides the SMWSQLStore3Table object that is * queried. * * @todo Maybe do something about redirects. The old code was * $oid = $this->store->smwIds->getSMWPageID($value->getDBkey(),$value->getNamespace(),$value->getInterwiki(),false); * * @param string $from * @param string $where * @param SMWSQLStore3Table $proptable * @param SMWDataItem $value * @param integer $tableindex */ protected function prepareValueQuery(&$from, &$where, $proptable, $value, $tableindex = 1) { $db = wfGetDB(DB_SLAVE); if ($value instanceof SMWDIContainer) { // recursive handling of containers $keys = array_keys($proptable->getFields($this->store)); $joinfield = "t{$tableindex}." . reset($keys); // this must be a type 'p' object $proptables = SMWSQLStore3::getPropertyTables(); $semanticData = $value->getSemanticData(); foreach ($semanticData->getProperties() as $subproperty) { $tableid = SMWSQLStore3::findPropertyTableID($subproperty); $subproptable = $proptables[$tableid]; foreach ($semanticData->getPropertyValues($subproperty) as $subvalue) { $tableindex++; if ($subproptable->idsubject) { // simply add property table to check values $from .= " INNER JOIN " . $db->tableName($subproptable->name) . " AS t{$tableindex} ON t{$tableindex}.s_id={$joinfield}"; } else { // exotic case with table that uses subject title+namespace in container object (should never happen in SMW core) $from .= " INNER JOIN " . $db->tableName('smw_ids') . " AS ids{$tableindex} ON ids{$tableindex}.smw_id={$joinfield}" . " INNER JOIN " . $db->tableName($subproptable->name) . " AS t{$tableindex} ON " . "t{$tableindex}.s_title=ids{$tableindex}.smw_title AND t{$tableindex}.s_namespace=ids{$tableindex}.smw_namespace"; } if ($subproptable->fixedproperty == false) { // the ID we get should be !=0, so no point in filtering the converse $where .= ($where ? ' AND ' : '') . "t{$tableindex}.p_id=" . $db->addQuotes($this->store->smwIds->getSMWPropertyID($subproperty)); } $this->prepareValueQuery($from, $where, $subproptable, $subvalue, $tableindex); } } } elseif (!is_null($value)) { // add conditions for given value $diHandler = $this->store->getDataItemHandlerForDIType($value->getDIType()); foreach ($diHandler->getWhereConds($value) as $fieldname => $value) { $where .= ($where ? ' AND ' : '') . "t{$tableindex}.{$fieldname}=" . $db->addQuotes($value); } } }
public function execute() { global $wgDBtype; if ($this->hasOption('setup')) { $store = new SMWSQLStore3(); // Lets do a drop to ensure the user doesn't has any Store3 tables already (happens when running this script twice) $tables = array('smw_stats'); foreach (SMWSQLStore3::getPropertyTables() as $proptable) { $tables[] = $proptable->name; } $dbw = wfGetDB(DB_MASTER); foreach ($tables as $table) { $name = $dbw->tableName($table); $dbw->query('DROP TABLE ' . ($wgDBtype == 'postgres' ? '' : 'IF EXISTS ') . $name, 'SMWMigrate::drop'); } $store->setup(); //enter user defined properties into smw_stats (internal ones are handled by setup already ) $query = 'Replace into ' . $dbw->tableName('smw_stats') . ' (pid,usage_count) Select smw_id,0 from ' . $dbw->tableName('smw_ids') . ' where smw_namespace = ' . SMW_NS_PROPERTY . ' and smw_iw = "" '; $dbw->query($query, 'SMWMigrate:commandLine'); } elseif ($this->hasOption('migrate')) { $options = array(); if ($this->hasArg(0)) { if ($this->hasArg(1)) { $options['LIMIT'] = $this->getArg(0); $options['OFFSET'] = $this->getArg(1); } } $dbw = wfGetDB(DB_MASTER); $oldStore = new SMWSQLStore2(); $newStore = new SMWSQLStore3(); $proptables = SMWSQLStore3::getPropertyTables(); //get properties $res = $dbw->select('smw_ids', array('smw_id', 'smw_title', 'smw_namespace'), array('smw_namespace' => SMW_NS_PROPERTY), __METHOD__, $options); foreach ($res as $row) { $property = new SMWDIProperty($row->smw_title); echo 'Now migrating data for Property ' . $property->getLabel() . " into Store3 \n"; //get the table $tableId = SMWSQLStore3::findPropertyTableID($property); $proptable = $proptables[$tableId]; //get the DIHandler $dataItemId = SMWDataValueFactory::getDataItemId($property->findPropertyTypeId()); $diHandler = $newStore->getDataItemHandlerForDIType($dataItemId); $subjects = $oldStore->getPropertySubjects($property, null); $insertions = array(); foreach ($subjects as $subject) { $sid = $newStore->makeSMWPageID($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $subject->getSubobjectName(), true, str_replace('_', ' ', $subject->getDBkey()) . $subject->getSubobjectName()); //now prepare udpates $propvals = $oldStore->getPropertyValues($subject, $property); $uvals = $proptable->idsubject ? array('s_id' => $sid) : array('s_title' => $subject->getDBkey(), 's_namespace' => $subject->getNamespace()); if ($proptable->fixedproperty == false) { $uvals['p_id'] = $newStore->makeSMWPropertyID($property); } foreach ($propvals as $propval) { $uvals = array_merge($uvals, $diHandler->getInsertValues($propval)); $insertions[] = $uvals; } } // now write to the DB for all subjects (is this too much?) $dbw->insert($proptable->name, $insertions, "SMW::migrate{$proptable->name}"); } $dbw->freeResult($res); } else { echo "Sorry I refuse to work without any options currently"; } }
/** * Using a preprocessed internal query description referenced by $rootid, * compute the proper result instance output for the given query. * @todo The SQL standard requires us to select all fields by which we sort, leading * to wrong results regarding the given limit: the user expects limit to be applied to * the number of distinct pages, but we can use DISTINCT only to whole rows. Thus, if * rows contain sortfields, then pages with multiple values for that field are distinct * and appear multiple times in the result. Filtering duplicates in post processing * would still allow such duplicates to push aside wanted values, leading to less than * "limit" results although there would have been "limit" really distinct results. For * this reason, we select sortfields only for POSTGRES. MySQL is able to perform what * we want here. It would be nice if we could eliminate the bug in POSTGRES as well. * * @param Query $query * @param integer $rootid * * @return QueryResult */ private function getInstanceQueryResult(Query $query, $rootid) { global $wgDBtype; $db = $this->store->getConnection(); $qobj = $this->querySegments[$rootid]; if ($qobj->joinfield === '') { // empty result, no query needed $result = new QueryResult($query->getDescription()->getPrintrequests(), $query, array(), $this->store, false); return $result; } $sql_options = $this->getSQLOptions($query, $rootid); // Selecting those is required in standard SQL (but MySQL does not require it). $sortfields = implode($qobj->sortfields, ','); $res = $db->select($db->tableName($qobj->joinTable) . " AS {$qobj->alias}" . $qobj->from, "DISTINCT {$qobj->alias}.smw_id AS id,{$qobj->alias}.smw_title AS t,{$qobj->alias}.smw_namespace AS ns,{$qobj->alias}.smw_iw AS iw,{$qobj->alias}.smw_subobject AS so,{$qobj->alias}.smw_sortkey AS sortkey" . ($wgDBtype == 'postgres' ? ($sortfields ? ',' : '') . $sortfields : ''), $qobj->where, __METHOD__, $sql_options); $qr = array(); $count = 0; // the number of fetched results ( != number of valid results in array $qr) $missedCount = 0; $dataItemCache = array(); $logToTable = array(); $hasFurtherResults = false; $prs = $query->getDescription()->getPrintrequests(); $diHandler = $this->store->getDataItemHandlerForDIType(DataItem::TYPE_WIKIPAGE); while ($count < $query->getLimit() && ($row = $db->fetchObject($res))) { if ($row->iw === '' || $row->iw[0] != ':') { // Catch exception for non-existing predefined properties that // still registered within non-updated pages (@see bug 48711) try { $dataItem = $diHandler->dataItemFromDBKeys(array($row->t, intval($row->ns), $row->iw, '', $row->so)); } catch (InvalidPredefinedPropertyException $e) { $logToTable[$row->t] = "issue creating a {$row->t} dataitem from a database row"; wfDebugLog('smw', __METHOD__ . ' ' . $e->getMessage() . "\n"); $dataItem = ''; } if ($dataItem instanceof DIWikiPage && !isset($dataItemCache[$dataItem->getHash()])) { $count++; $dataItemCache[$dataItem->getHash()] = true; $qr[] = $dataItem; // These IDs are usually needed for displaying the page (esp. if more property values are displayed): $this->store->smwIds->setCache($row->t, $row->ns, $row->iw, $row->so, $row->id, $row->sortkey); } else { $missedCount++; $logToTable[$row->t] = "skip result for {$row->t} existing cache entry / query " . $query->getHash(); } } else { $missedCount++; $logToTable[$row->t] = "skip result for {$row->t} due to an internal `{$row->iw}` pointer / query " . $query->getHash(); } } if ($db->fetchObject($res)) { $count++; } if ($logToTable !== array()) { wfDebugLog('smw', __METHOD__ . ' ' . implode(',', $logToTable) . "\n"); } if ($count > $query->getLimit() || $count + $missedCount > $query->getLimit()) { $hasFurtherResults = true; } $db->freeResult($res); $result = new QueryResult($prs, $query, $qr, $this->store, $hasFurtherResults); return $result; }
/** * Method to return the fields for this table * * @since 1.8 * * @param SMWSQLStore3 $store * * @return array */ public function getFields(\SMWSQLStore3 $store) { $diHandler = $store->getDataItemHandlerForDIType($this->diType); return $diHandler->getTableFields(); }