/**
* Sets up the property tables.
*
* @since 1.8
* @param array $dbtypes
* @param DatabaseBase|Database $db
* @param SMWSQLStore3SetupHandlers|null $reportTo
*/
protected function setupPropertyTables(array $dbtypes, $db, SMWSQLStore3SetupHandlers $reportTo = null)
{
$addedCustomTypeSignatures = false;
foreach (SMWSQLStore3::getPropertyTables() as $proptable) {
$diHandler = $this->store->getDataItemHandlerForDIType($proptable->getDiType());
// Prepare indexes. By default, property-value tables
// have the following indexes:
//
// sp: getPropertyValues(), getSemanticData(), getProperties()
// po: ask, getPropertySubjects()
//
// The "p" component is omitted for tables with fixed property.
$indexes = array();
if ($proptable->usesIdSubject()) {
$fieldarray = array('s_id' => $dbtypes['p'] . ' NOT NULL');
$indexes['sp'] = 's_id';
} else {
$fieldarray = array('s_title' => $dbtypes['t'] . ' NOT NULL', 's_namespace' => $dbtypes['n'] . ' NOT NULL');
$indexes['sp'] = 's_title,s_namespace';
}
$indexes['po'] = $diHandler->getIndexField();
if (!$proptable->isFixedPropertyTable()) {
$fieldarray['p_id'] = $dbtypes['p'] . ' NOT NULL';
$indexes['po'] = 'p_id,' . $indexes['po'];
$indexes['sp'] = $indexes['sp'] . ',p_id';
}
// TODO Special handling; concepts should be handled differently
// in the future. See comments in SMW_DIHandler_Concept.php.
if ($proptable->getDiType() == SMWDataItem::TYPE_CONCEPT) {
unset($indexes['po']);
}
$indexes = array_merge($indexes, $diHandler->getTableIndexes());
$indexes = array_unique($indexes);
foreach ($diHandler->getTableFields() as $fieldname => $typeid) {
// If the type signature is not recognized and the custom signatures have not been added, add them.
if (!$addedCustomTypeSignatures && !array_key_exists($typeid, $dbtypes)) {
wfRunHooks('SMWCustomSQLStoreFieldType', array(&$dbtypes));
$addedCustomTypeSignatures = true;
}
// Only add the type when the signature was recognized, otherwise ignore it silently.
if (array_key_exists($typeid, $dbtypes)) {
$fieldarray[$fieldname] = $dbtypes[$typeid];
}
}
SMWSQLHelpers::setupTable($proptable->getName(), $fieldarray, $db, $reportTo);
SMWSQLHelpers::setupIndex($proptable->getName(), $indexes, $db, $reportTo);
}
}
/**
* Get the array of all stored values for some property.
*
* @since 1.8
*
* @param DIProperty $property
*
* @return array of SMWDataItem
*/
public function getPropertyValues(DIProperty $property)
{
if ($property->isInverse()) {
// we never have any data for inverses
return array();
}
if (array_key_exists($property->getKey(), $this->mStubPropVals)) {
// Not catching exception here; the
$this->unstubProperty($property->getKey(), $property);
$propertyTypeId = $property->findPropertyTypeID();
$propertyDiId = DataTypeRegistry::getInstance()->getDataItemId($propertyTypeId);
foreach ($this->mStubPropVals[$property->getKey()] as $dbkeys) {
try {
$diHandler = $this->store->getDataItemHandlerForDIType($propertyDiId);
$di = $diHandler->dataItemFromDBKeys($dbkeys);
if ($this->mNoDuplicates) {
$this->mPropVals[$property->getKey()][$di->getHash()] = $di;
} else {
$this->mPropVals[$property->getKey()][] = $di;
}
} catch (SMWDataItemException $e) {
// ignore data
}
}
unset($this->mStubPropVals[$property->getKey()]);
}
return parent::getPropertyValues($property);
}
/**
* Method to get all subobjects for a given subject.
*
* @since 1.8
* @param SMWDIWikiPage $subject
*
* @return array of smw_id => SMWDIWikiPage
*/
protected function getSubobjects(SMWDIWikiPage $subject)
{
$db = $this->store->getConnection();
$res = $db->select($db->tablename(SMWSql3SmwIds::tableName), 'smw_id,smw_subobject,smw_sortkey', 'smw_title = ' . $db->addQuotes($subject->getDBkey()) . ' AND ' . 'smw_namespace = ' . $db->addQuotes($subject->getNamespace()) . ' AND ' . 'smw_iw = ' . $db->addQuotes($subject->getInterwiki()) . ' AND ' . 'smw_subobject != ' . $db->addQuotes(''), __METHOD__);
$diHandler = $this->store->getDataItemHandlerForDIType(SMWDataItem::TYPE_WIKIPAGE);
$subobjects = array();
foreach ($res as $row) {
$subobjects[$row->smw_id] = $diHandler->dataItemFromDBKeys(array($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $row->smw_sortkey, $row->smw_subobject));
}
$db->freeResult($res);
return $subobjects;
}
/**
* Extend the given update array to account for the data in the
* SMWSemanticData object. The subject page of the data container is
* ignored, and the given $sid (subject page id) is used directly. If
* this ID is 0, then $subject is used to find an ID. This is usually
* the case for all internal objects (subobjects) that are created in
* writing sub-SemanticData.
*
* The function returns the id that was used for writing. Especially,
* any newly created internal id is returned.
*
* @param $updates array
* @param $data SMWSemanticData
* @param $sid integer pre-computed id if available or 0 if ID should be sought
* @param $subject SMWDIWikiPage subject to which the data refers
*/
protected function prepareDBUpdates(&$updates, SMWSemanticData $data, $sid, SMWDIWikiPage $subject)
{
if ($sid == 0) {
$sid = $this->store->smwIds->makeSMWPageID($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $subject->getSubobjectName(), true, str_replace('_', ' ', $subject->getDBkey()) . $subject->getSubobjectName());
}
$proptables = SMWSQLStore3::getPropertyTables();
foreach ($data->getProperties() as $property) {
if ($property->getKey() == '_SKEY' || $property->getKey() == '_REDI') {
continue;
// skip these here, we store them differently
}
$tableid = SMWSQLStore3::findPropertyTableID($property);
$proptable = $proptables[$tableid];
///TODO check needed if subject is null (would happen if a user defined proptable with !idsubject was used on an internal object -- currently this is not possible
$uvals = $proptable->idsubject ? array('s_id' => $sid) : array('s_title' => $subject->getDBkey(), 's_namespace' => $subject->getNamespace());
if ($proptable->fixedproperty == false) {
$uvals['p_id'] = $this->store->smwIds->makeSMWPropertyID($property);
}
foreach ($data->getPropertyValues($property) as $di) {
if ($di instanceof SMWDIError) {
// error values, ignore
continue;
}
$diHandler = $this->store->getDataItemHandlerForDIType($di->getDIType());
$uvals = array_merge($uvals, $diHandler->getInsertValues($di));
if (!array_key_exists($proptable->name, $updates)) {
$updates[$proptable->name] = array();
}
$updates[$proptable->name][] = $uvals;
}
}
// Special handling of Concepts
if ($subject->getNamespace() == SMW_NS_CONCEPT && $subject->getSubobjectName() == '') {
if (array_key_exists('smw_fpt_conc', $updates) && count($updates['smw_fpt_conc']) != 0) {
$updates['smw_fpt_conc'] = end($updates['smw_fpt_conc']);
unset($updates['smw_fpt_conc']['cache_date']);
unset($updates['smw_fpt_conc']['cache_count']);
} else {
$updates['smw_fpt_conc'] = array('concept_txt' => '', 'concept_docu' => '', 'concept_features' => 0, 'concept_size' => -1, 'concept_depth' => -1);
}
}
return $sid;
}
/**
* Create an array of rows to insert into property tables in order to
* store the given SMWSemanticData. The given $sid (subject page id) is
* used directly and must belong to the subject of the data container.
* Sortkeys are ignored since they are not stored in a property table
* but in the ID table.
*
* The returned array uses property table names as keys and arrays of
* table rows as values. Each table row is an array mapping column
* names to values.
*
* @note Property tables that do not use ids as subjects are ignored.
* This just excludes redirects that are handled differently anyway;
* it would not make a difference to include them here.
*
* @since 1.8
* @param integer $sid
* @param SMWSemanticData $data
* @param DatabaseBase $dbr used for reading only
* @return array
*/
protected function preparePropertyTableInserts($sid, SMWSemanticData $data, DatabaseBase $dbr)
{
$updates = array();
$subject = $data->getSubject();
$propertyTables = SMWSQLStore3::getPropertyTables();
foreach ($data->getProperties() as $property) {
$tableId = SMWSQLStore3::findPropertyTableID($property);
if (is_null($tableId)) {
// not stored in a property table, e.g., sortkeys
continue;
}
$propertyTable = $propertyTables[$tableId];
if (!$propertyTable->usesIdSubject()) {
// not using subject ids, e.g., redirects
continue;
}
$insertValues = array('s_id' => $sid);
if (!$propertyTable->isFixedPropertyTable()) {
$insertValues['p_id'] = $this->store->smwIds->makeSMWPropertyID($property);
}
foreach ($data->getPropertyValues($property) as $di) {
if ($di instanceof SMWDIError) {
// ignore error values
continue;
}
if (!array_key_exists($propertyTable->getName(), $updates)) {
$updates[$propertyTable->getName()] = array();
}
$diHandler = $this->store->getDataItemHandlerForDIType($di->getDIType());
// Note that array_merge creates a new array; not overwriting past entries here
$insertValues = array_merge($insertValues, $diHandler->getInsertValues($di));
$insertValueKey = self::makeDatabaseRowKey($insertValues);
$updates[$propertyTable->getName()][$insertValueKey] = $insertValues;
}
}
// Special handling of Concepts
if ($subject->getNamespace() == SMW_NS_CONCEPT && $subject->getSubobjectName() == '') {
$this->prepareConceptTableInserts($sid, $updates, $dbr);
}
return $updates;
}
/**
* Helper function to compute from and where strings for a DB query so that
* only rows of the given value object match. The parameter $tableindex
* counts that tables used in the query to avoid duplicate table names. The
* parameter $proptable provides the SMWSQLStore3Table object that is
* queried.
*
* @todo Maybe do something about redirects. The old code was
* $oid = $this->store->smwIds->getSMWPageID($value->getDBkey(),$value->getNamespace(),$value->getInterwiki(),false);
*
* @param string $from
* @param string $where
* @param SMWSQLStore3Table $proptable
* @param SMWDataItem $value
* @param integer $tableindex
*/
protected function prepareValueQuery(&$from, &$where, $proptable, $value, $tableindex = 1)
{
$db = wfGetDB(DB_SLAVE);
if ($value instanceof SMWDIContainer) {
// recursive handling of containers
$keys = array_keys($proptable->getFields($this->store));
$joinfield = "t{$tableindex}." . reset($keys);
// this must be a type 'p' object
$proptables = SMWSQLStore3::getPropertyTables();
$semanticData = $value->getSemanticData();
foreach ($semanticData->getProperties() as $subproperty) {
$tableid = SMWSQLStore3::findPropertyTableID($subproperty);
$subproptable = $proptables[$tableid];
foreach ($semanticData->getPropertyValues($subproperty) as $subvalue) {
$tableindex++;
if ($subproptable->idsubject) {
// simply add property table to check values
$from .= " INNER JOIN " . $db->tableName($subproptable->name) . " AS t{$tableindex} ON t{$tableindex}.s_id={$joinfield}";
} else {
// exotic case with table that uses subject title+namespace in container object (should never happen in SMW core)
$from .= " INNER JOIN " . $db->tableName('smw_ids') . " AS ids{$tableindex} ON ids{$tableindex}.smw_id={$joinfield}" . " INNER JOIN " . $db->tableName($subproptable->name) . " AS t{$tableindex} ON " . "t{$tableindex}.s_title=ids{$tableindex}.smw_title AND t{$tableindex}.s_namespace=ids{$tableindex}.smw_namespace";
}
if ($subproptable->fixedproperty == false) {
// the ID we get should be !=0, so no point in filtering the converse
$where .= ($where ? ' AND ' : '') . "t{$tableindex}.p_id=" . $db->addQuotes($this->store->smwIds->getSMWPropertyID($subproperty));
}
$this->prepareValueQuery($from, $where, $subproptable, $subvalue, $tableindex);
}
}
} elseif (!is_null($value)) {
// add conditions for given value
$diHandler = $this->store->getDataItemHandlerForDIType($value->getDIType());
foreach ($diHandler->getWhereConds($value) as $fieldname => $value) {
$where .= ($where ? ' AND ' : '') . "t{$tableindex}.{$fieldname}=" . $db->addQuotes($value);
}
}
}
public function execute()
{
global $wgDBtype;
if ($this->hasOption('setup')) {
$store = new SMWSQLStore3();
// Lets do a drop to ensure the user doesn't has any Store3 tables already (happens when running this script twice)
$tables = array('smw_stats');
foreach (SMWSQLStore3::getPropertyTables() as $proptable) {
$tables[] = $proptable->name;
}
$dbw = wfGetDB(DB_MASTER);
foreach ($tables as $table) {
$name = $dbw->tableName($table);
$dbw->query('DROP TABLE ' . ($wgDBtype == 'postgres' ? '' : 'IF EXISTS ') . $name, 'SMWMigrate::drop');
}
$store->setup();
//enter user defined properties into smw_stats (internal ones are handled by setup already )
$query = 'Replace into ' . $dbw->tableName('smw_stats') . ' (pid,usage_count) Select smw_id,0 from ' . $dbw->tableName('smw_ids') . ' where smw_namespace = ' . SMW_NS_PROPERTY . ' and smw_iw = "" ';
$dbw->query($query, 'SMWMigrate:commandLine');
} elseif ($this->hasOption('migrate')) {
$options = array();
if ($this->hasArg(0)) {
if ($this->hasArg(1)) {
$options['LIMIT'] = $this->getArg(0);
$options['OFFSET'] = $this->getArg(1);
}
}
$dbw = wfGetDB(DB_MASTER);
$oldStore = new SMWSQLStore2();
$newStore = new SMWSQLStore3();
$proptables = SMWSQLStore3::getPropertyTables();
//get properties
$res = $dbw->select('smw_ids', array('smw_id', 'smw_title', 'smw_namespace'), array('smw_namespace' => SMW_NS_PROPERTY), __METHOD__, $options);
foreach ($res as $row) {
$property = new SMWDIProperty($row->smw_title);
echo 'Now migrating data for Property ' . $property->getLabel() . " into Store3 \n";
//get the table
$tableId = SMWSQLStore3::findPropertyTableID($property);
$proptable = $proptables[$tableId];
//get the DIHandler
$dataItemId = SMWDataValueFactory::getDataItemId($property->findPropertyTypeId());
$diHandler = $newStore->getDataItemHandlerForDIType($dataItemId);
$subjects = $oldStore->getPropertySubjects($property, null);
$insertions = array();
foreach ($subjects as $subject) {
$sid = $newStore->makeSMWPageID($subject->getDBkey(), $subject->getNamespace(), $subject->getInterwiki(), $subject->getSubobjectName(), true, str_replace('_', ' ', $subject->getDBkey()) . $subject->getSubobjectName());
//now prepare udpates
$propvals = $oldStore->getPropertyValues($subject, $property);
$uvals = $proptable->idsubject ? array('s_id' => $sid) : array('s_title' => $subject->getDBkey(), 's_namespace' => $subject->getNamespace());
if ($proptable->fixedproperty == false) {
$uvals['p_id'] = $newStore->makeSMWPropertyID($property);
}
foreach ($propvals as $propval) {
$uvals = array_merge($uvals, $diHandler->getInsertValues($propval));
$insertions[] = $uvals;
}
}
// now write to the DB for all subjects (is this too much?)
$dbw->insert($proptable->name, $insertions, "SMW::migrate{$proptable->name}");
}
$dbw->freeResult($res);
} else {
echo "Sorry I refuse to work without any options currently";
}
}
/**
* Using a preprocessed internal query description referenced by $rootid,
* compute the proper result instance output for the given query.
* @todo The SQL standard requires us to select all fields by which we sort, leading
* to wrong results regarding the given limit: the user expects limit to be applied to
* the number of distinct pages, but we can use DISTINCT only to whole rows. Thus, if
* rows contain sortfields, then pages with multiple values for that field are distinct
* and appear multiple times in the result. Filtering duplicates in post processing
* would still allow such duplicates to push aside wanted values, leading to less than
* "limit" results although there would have been "limit" really distinct results. For
* this reason, we select sortfields only for POSTGRES. MySQL is able to perform what
* we want here. It would be nice if we could eliminate the bug in POSTGRES as well.
*
* @param Query $query
* @param integer $rootid
*
* @return QueryResult
*/
private function getInstanceQueryResult(Query $query, $rootid)
{
global $wgDBtype;
$db = $this->store->getConnection();
$qobj = $this->querySegments[$rootid];
if ($qobj->joinfield === '') {
// empty result, no query needed
$result = new QueryResult($query->getDescription()->getPrintrequests(), $query, array(), $this->store, false);
return $result;
}
$sql_options = $this->getSQLOptions($query, $rootid);
// Selecting those is required in standard SQL (but MySQL does not require it).
$sortfields = implode($qobj->sortfields, ',');
$res = $db->select($db->tableName($qobj->joinTable) . " AS {$qobj->alias}" . $qobj->from, "DISTINCT {$qobj->alias}.smw_id AS id,{$qobj->alias}.smw_title AS t,{$qobj->alias}.smw_namespace AS ns,{$qobj->alias}.smw_iw AS iw,{$qobj->alias}.smw_subobject AS so,{$qobj->alias}.smw_sortkey AS sortkey" . ($wgDBtype == 'postgres' ? ($sortfields ? ',' : '') . $sortfields : ''), $qobj->where, __METHOD__, $sql_options);
$qr = array();
$count = 0;
// the number of fetched results ( != number of valid results in array $qr)
$missedCount = 0;
$dataItemCache = array();
$logToTable = array();
$hasFurtherResults = false;
$prs = $query->getDescription()->getPrintrequests();
$diHandler = $this->store->getDataItemHandlerForDIType(DataItem::TYPE_WIKIPAGE);
while ($count < $query->getLimit() && ($row = $db->fetchObject($res))) {
if ($row->iw === '' || $row->iw[0] != ':') {
// Catch exception for non-existing predefined properties that
// still registered within non-updated pages (@see bug 48711)
try {
$dataItem = $diHandler->dataItemFromDBKeys(array($row->t, intval($row->ns), $row->iw, '', $row->so));
} catch (InvalidPredefinedPropertyException $e) {
$logToTable[$row->t] = "issue creating a {$row->t} dataitem from a database row";
wfDebugLog('smw', __METHOD__ . ' ' . $e->getMessage() . "\n");
$dataItem = '';
}
if ($dataItem instanceof DIWikiPage && !isset($dataItemCache[$dataItem->getHash()])) {
$count++;
$dataItemCache[$dataItem->getHash()] = true;
$qr[] = $dataItem;
// These IDs are usually needed for displaying the page (esp. if more property values are displayed):
$this->store->smwIds->setCache($row->t, $row->ns, $row->iw, $row->so, $row->id, $row->sortkey);
} else {
$missedCount++;
$logToTable[$row->t] = "skip result for {$row->t} existing cache entry / query " . $query->getHash();
}
} else {
$missedCount++;
$logToTable[$row->t] = "skip result for {$row->t} due to an internal `{$row->iw}` pointer / query " . $query->getHash();
}
}
if ($db->fetchObject($res)) {
$count++;
}
if ($logToTable !== array()) {
wfDebugLog('smw', __METHOD__ . ' ' . implode(',', $logToTable) . "\n");
}
if ($count > $query->getLimit() || $count + $missedCount > $query->getLimit()) {
$hasFurtherResults = true;
}
$db->freeResult($res);
$result = new QueryResult($prs, $query, $qr, $this->store, $hasFurtherResults);
return $result;
}
/**
* Method to return the fields for this table
*
* @since 1.8
*
* @param SMWSQLStore3 $store
*
* @return array
*/
public function getFields(\SMWSQLStore3 $store)
{
$diHandler = $store->getDataItemHandlerForDIType($this->diType);
return $diHandler->getTableFields();
}
