protected function prepareColumnsMetaDataProperties(DataSourceMetaData $datasource, array $tableNames)
{
$datasourceHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
$sql = 'SELECT c.relname AS ' . self::PROPERTY__TABLE_NAME . ', ' . ' a.attname AS ' . self::PROPERTY__COLUMN_NAME . ', ' . ' a.attnum AS ' . self::PROPERTY__COLUMN_INDEX . ', ' . ' t.typname AS ' . self::PROPERTY__COLUMN_TYPE . ' FROM pg_class c INNER JOIN pg_namespace ns ON ns.oid = c.relnamespace' . ' INNER JOIN pg_attribute a ON a.attrelid = c.oid' . ' INNER JOIN pg_type t ON t.oid = a.atttypid' . " WHERE c.relname IN ('" . implode("', '", $tableNames) . "')" . " AND c.relkind IN ('r','v')" . " AND ns.nspname = '{$datasource->schema}'" . ' AND a.attnum > 0';
LogHelper::log_info(new StatementLogMessage('metadata.dataset.systemTable', $sql));
return $datasourceHandler->executeQuery(new DataControllerCallContext(), $datasource, $sql, new PassthroughResultFormatter());
}
public function check(DataSourceMetaData $datasourceA, DataSourceMetaData $datasourceB)
{
// if the two objects point to the same configuration
$isDataSourceCompatible = $datasourceA->name == $datasourceB->name;
$isTypeCompatible = $isHostCompatible = $isDatabaseCompatible = $isDataSourceCompatible;
if (!$isDataSourceCompatible) {
$datasourceHandlerA = DataSourceQueryFactory::getInstance()->getHandler($datasourceA->type);
$datasourceHandlerB = DataSourceQueryFactory::getInstance()->getHandler($datasourceB->type);
// checking if handler type is the same
$isTypeCompatible = $datasourceA->type == $datasourceB->type;
// checking if handler type is different but database type is the same
if (!$isTypeCompatible) {
$isTypeCompatible = $datasourceHandlerA->getDataSourceType() == $datasourceHandlerB->getDataSourceType();
}
if ($isTypeCompatible) {
$isDataSourceCompatible = $isTypeCompatible;
// if host name is present for both data sources we compare them. Otherwise just ignoring
$isHostCompatible = isset($datasourceA->host) && isset($datasourceB->host) ? $datasourceA->host == $datasourceB->host : TRUE;
// if port number is present for both data sources we compare them. Otherwise just ignoring
if ($isHostCompatible && isset($datasourceA->port) && isset($datasourceB->port)) {
$isHostCompatible = $datasourceA->port == $datasourceB->port;
}
if ($isHostCompatible) {
// if database name is present for both data sources we compare them. Otherwise just ignoring
$isDatabaseCompatible = isset($datasourceA->database) && isset($datasourceB->database) ? $datasourceA->database == $datasourceB->database : TRUE;
}
}
}
return array($isDataSourceCompatible, $isTypeCompatible, $isHostCompatible, $isDatabaseCompatible);
}
/**
* @static
* @return DataSourceQueryFactory
*/
public static function getInstance()
{
if (!isset(self::$factory)) {
self::$factory = new DataSourceQueryFactory();
}
return self::$factory;
}
protected function executeQuery(DataSourceMetaData $datasource, $operationName, $sql) {
LogHelper::log_info(new StatementLogMessage("metamodel.system.{$operationName}[{$datasource->type}][{$datasource->name}]", $sql));
$executionCallContext = new DataControllerCallContext();
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
return $datasourceQueryHandler->executeQuery($executionCallContext, $datasource, $sql);
}
protected function generateColumnApplicationType(SystemTableMetaModelLoaderCallContext $callcontext, DataSourceMetaData $datasource, ColumnMetaData $column) {
$classcontext = $callcontext->getClassContext($this);
if (isset($classcontext->statementExecutionCallback[$datasource->type])) {
$statementExecutionCallback = $classcontext->statementExecutionCallback[$datasource->type];
}
else {
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
$statementExecutionCallback = $datasourceQueryHandler->prepareQueryStatementExecutionCallbackInstance();
$classcontext->statementExecutionCallback[$datasource->type] = $statementExecutionCallback;
}
$statementExecutionCallback->generateColumnType($column);
}
public function dropColumnStorage(DataControllerCallContext $callcontext, DataSourceStructureHandler $datasourceStructureHandler, DatasetMetaData $dataset, $columnName, $stage) {
if ($stage == DatasetStorageObserver::STAGE__AFTER) {
$environment_metamodel = data_controller_get_environment_metamodel();
$originalLogicalColumn = $this->originalLogicalDataset->getColumn($columnName);
$originalColumn = $dataset->getColumn($columnName);
$datasource = $environment_metamodel->getDataSource($dataset->datasourceName);
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
$generator = new FunctionDateDimensionMetaDataGenerator($datasourceQueryHandler);
$generator->clean($originalColumn);
$generator->clean($originalLogicalColumn);
}
parent::dropColumnStorage($callcontext, $datasourceStructureHandler, $dataset, $columnName, $stage);
}
public function finalize(AbstractMetaModel $metamodel) {
parent::finalize($metamodel);
$environment_metamodel = data_controller_get_environment_metamodel();
$generators = NULL;
foreach ($metamodel->datasets as $dataset) {
$datasource = $environment_metamodel->getDataSource($dataset->datasourceName);
// preparing generator for the data source type
if (!isset($generators[$datasource->type])) {
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
$generators[$datasource->type] = new FunctionDateDimensionMetaDataGenerator($datasourceQueryHandler);
}
$generator = $generators[$datasource->type];
foreach ($dataset->columns as $column) {
$generator->generate($column);
}
}
}
public function load(AbstractMetaModelFactory $factory, AbstractMetaModel $metamodel, array $filters = NULL, $finalAttempt)
{
$selectedDataSourceType = $this->selectedDataSourceType();
LogHelper::log_notice(t("Finalizing Dataset Meta Data for '@databaseType' database connections ...", array('@databaseType' => $selectedDataSourceType)));
if ($finalAttempt === FALSE) {
return self::LOAD_STATE__POSTPONED;
}
$finalizedDatasetCount = 0;
$environment_metamodel = data_controller_get_environment_metamodel();
// processing all database connections
foreach ($environment_metamodel->datasources as $datasource) {
if ($datasource->type !== $selectedDataSourceType) {
continue;
}
// selecting datasets which could be processed for the selected connection
$selectedSources = NULL;
foreach ($metamodel->datasets as $dataset) {
// the dataset should belong to the selected data source
if ($dataset->datasourceName !== $datasource->name) {
continue;
}
// the dataset has to be of type table
if (DatasetTypeHelper::detectDatasetSourceType($dataset) !== DatasetTypeHelper::DATASET_SOURCE_TYPE__TABLE) {
continue;
}
// whole dataset meta data was prepared using different method. There is nothing else can be done
if ($dataset->isComplete()) {
continue;
}
$tableName = strtolower($dataset->source);
// invalidating existing column indexes
$dataset->invalidateColumnIndexes();
// there could be several datasets for one table
$selectedSources[$tableName][] = $dataset;
}
if (!isset($selectedSources)) {
continue;
}
$datasourceHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
$metadataCallback = $datasourceHandler->prepareQueryStatementExecutionCallbackInstance();
// processing meta data for selected datasets
$columnsMetaDataProperties = $this->prepareColumnsMetaDataProperties($datasource, array_keys($selectedSources));
if (isset($columnsMetaDataProperties)) {
foreach ($columnsMetaDataProperties as $columnMetaDataProperties) {
$tableName = strtolower($columnMetaDataProperties[self::PROPERTY__TABLE_NAME]);
$datasets = $selectedSources[$tableName];
foreach ($datasets as $dataset) {
$column = new ColumnMetaData();
$column->name = strtolower($columnMetaDataProperties[self::PROPERTY__COLUMN_NAME]);
$column->columnIndex = $columnMetaDataProperties[self::PROPERTY__COLUMN_INDEX];
// preparing column type
$column->type->databaseType = $columnMetaDataProperties[self::PROPERTY__COLUMN_TYPE];
$column->type->applicationType = $metadataCallback->calculateApplicationDataType($column);
// checking if the column is a system column which should be invisible
if (substr_compare($column->name, DatasetSystemColumnNames::COLUMN_NAME_PREFIX, 0, strlen(DatasetSystemColumnNames::COLUMN_NAME_PREFIX)) === 0) {
$column->visible = FALSE;
}
$dataset->initializeColumnFrom($column);
}
}
}
// marking all selected datasets as completed
foreach ($selectedSources as $datasets) {
foreach ($datasets as $dataset) {
$dataset->markAsComplete();
$finalizedDatasetCount++;
}
}
}
LogHelper::log_info(t('Finalized @datasetCount dataset meta data configurations', array('@datasetCount' => $finalizedDatasetCount)));
return self::LOAD_STATE__SUCCESSFUL;
}
public function insertOrUpdateOrDeleteDatasetRecords(DataControllerCallContext $callcontext, AbstractDatasetManipulationRequest $request) {
$environment_metamodel = data_controller_get_environment_metamodel();
$metamodel = data_controller_get_metamodel();
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($this->getDataSourceType());
$dataset = $metamodel->getDataset($request->datasetName);
$datasource = $environment_metamodel->getDataSource($dataset->datasourceName);
$recordMetaData = isset($request->recordsHolder->recordMetaData) ? $request->recordsHolder->recordMetaData : $dataset;
$isRecordIndexed = $request->recordsHolder instanceof IndexedRecordsHolder;
$keyColumnNames = $recordMetaData->getKeyColumnNames();
$nonkeyColumnNames = $recordMetaData->findNonKeyColumnNames();
// preparing a request to find existing records
$queryRequest = new DatasetQueryRequest($request->datasetName);
// loading only key columns from database
$queryRequest->addColumns($keyColumnNames);
// a request can be more efficient for single column key
if (count($keyColumnNames) == 1) {
list($keyColumnIndex, $keyColumnName) = each($keyColumnNames);
$keyColumnIdentifier = $isRecordIndexed ? $keyColumnIndex : $keyColumnName;
$keyValues = NULL;
foreach ($request->recordsHolder->records as $record) {
ArrayHelper::addUniqueValue($keyValues, $record->getColumnValue($keyColumnIdentifier, TRUE));
}
$queryRequest->addQueryValue(
0,
$keyColumnName, OperatorFactory::getInstance()->initiateHandler(EqualOperatorHandler::OPERATOR__NAME, array($keyValues)));
}
else {
for ($i = 0, $count = count($request->recordsHolder->records); $i < $count; $i++) {
$record = $request->recordsHolder->records[$i];
foreach ($keyColumnNames as $keyColumnIndex => $keyColumnName) {
$keyColumnIdentifier = $isRecordIndexed ? $keyColumnIndex : $keyColumnName;
$keyColumnValue = $record->getColumnValue($keyColumnIdentifier, TRUE);
$queryRequest->addQueryValue(
$i,
$keyColumnName, OperatorFactory::getInstance()->initiateHandler(EqualOperatorHandler::OPERATOR__NAME, $keyColumnValue));
}
}
}
// loading existing records ... if any
$existingRecordFormatter = new QueryKeyResultFormatter($keyColumnNames);
$existingRecords = $existingRecordFormatter->formatRecords($datasourceQueryHandler->queryDataset($callcontext, $queryRequest));
// sorting out records for insert, update and delete operations
$keyedRecords = $insertedRecordKeys = $updatedRecordKeys = $deletedRecordKeys = NULL;
foreach ($request->recordsHolder->records as $record) {
$keyParts = NULL;
foreach ($keyColumnNames as $keyColumnIndex => $keyColumnName) {
$keyColumnIdentifier = $isRecordIndexed ? $keyColumnIndex : $keyColumnName;
$keyParts[] = $record->getColumnValue($keyColumnIdentifier, TRUE);
}
$key = ArrayHelper::prepareCompositeKey($keyParts);
$keyedRecords[$key] = $record;
// checking if the record has to be deleted
$isDeletable = TRUE;
if (isset($nonkeyColumnNames)) {
foreach ($nonkeyColumnNames as $columnIndex => $columnName) {
$columnIdentifier = $isRecordIndexed ? $columnIndex : $columnName;
if ($record->getColumnValue($columnIdentifier) != NULL) {
$isDeletable = FALSE;
break;
}
}
}
else {
// the dataset has NO non-key columns. We should not delete these records
$isDeletable = FALSE;
}
if ($isDeletable) {
unset($insertedRecordKeys[$key]);
unset($updatedRecordKeys[$key]);
// the record physically present in database and needs to be deleted
if (isset($existingRecords[$key])) {
unset($existingRecords[$key]);
$deletedRecordKeys[$key] = TRUE;
}
}
elseif (isset($insertedRecordKeys[$key])) {
// the key has been already used to insert a record within this batch. This record needs to be part of update operation
$updatedRecordKeys[$key] = TRUE;
}
elseif (isset($existingRecords[$key])) {
$updatedRecordKeys[$key] = TRUE;
}
else {
$insertedRecordKeys[$key] = TRUE;
}
}
$sqls = NULL;
// deleting existing records
$deletedRecordCount = 0;
if (isset($deletedRecordKeys)) {
$deleteRecordHolder = $this->prepareRecordHolder($request, $keyedRecords, $deletedRecordKeys);
// preparing request
$deleteRequest = new DatasetDeleteRequest($request->datasetName, $deleteRecordHolder);
// preparing statements to delete records from the database
ArrayHelper::appendValue($sqls, $this->prepareDeleteDatasetRecordStatements($callcontext, $deleteRequest));
$deletedRecordCount = count($deleteRecordHolder->records);
}
// inserting new records
$insertedRecordCount = 0;
if (isset($insertedRecordKeys)) {
$insertRecordHolder = $this->prepareRecordHolder($request, $keyedRecords, $insertedRecordKeys);
// preparing request
$insertRequest = new DatasetInsertRequest($request->datasetName, $insertRecordHolder);
// preparing statements to insert records into the database
ArrayHelper::appendValue($sqls, $this->prepareInsertDatasetRecordStatements($callcontext, $insertRequest));
$insertedRecordCount = count($insertRecordHolder->records);
}
// updating existing records
$updatedRecordCount = 0;
if (isset($updatedRecordKeys)) {
$updateRecordHolder = $this->prepareRecordHolder($request, $keyedRecords, $updatedRecordKeys);
// preparing request
$updateRequest = new DatasetUpdateRequest($request->datasetName, $updateRecordHolder);
// preparing statements to update records in the database
ArrayHelper::appendValue($sqls, $this->prepareUpdateDatasetRecordStatements($callcontext, $updateRequest));
$updatedRecordCount = count($updateRecordHolder->records);
}
$affectedRecordCount = isset($sqls) ? $this->executeManipulationStatementBatch($datasource, $sqls) : 0;
if (($insertedRecordCount + $updatedRecordCount + $deletedRecordCount) < $affectedRecordCount) {
throw new IllegalStateException(t('Number of affected records is greater than expected number of inserted, updated and deleted records'));
}
return array($insertedRecordCount, $updatedRecordCount, $deletedRecordCount);
}
protected function assembleForeignKeyConstraints(DataSourceHandler $handler, DatasetMetaData $dataset, $indent, &$sql)
{
$metamodel = data_controller_get_metamodel();
foreach ($dataset->getColumns() as $column) {
$columnName = $column->name;
if (!isset($column->type->sourceApplicationType)) {
continue;
}
// the column has to contain a reference to another dataset
$dimensionLookupHandler = DimensionLookupFactory::getInstance()->getHandler($column->type->sourceApplicationType);
list($referencedDatasetName) = $dimensionLookupHandler->adjustReferencePointColumn($metamodel, $dataset->name, $column->name);
if ($dataset->name == $referencedDatasetName) {
continue;
}
$referencedDataset = $metamodel->getDataset($referencedDatasetName);
// we can create a foreign key constraint referenced to a table only
$referencedDatasetSourceType = DatasetTypeHelper::detectDatasetSourceType($referencedDataset);
if ($referencedDatasetSourceType != DatasetTypeHelper::DATASET_SOURCE_TYPE__TABLE) {
continue;
}
$referencedOwner = NULL;
if ($dataset->datasourceName != $referencedDataset->datasourceName) {
// if we cannot join datasets we cannot create a foreign key constraint
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($handler->getDataSourceType());
if (!$datasourceQueryHandler->isJoinSupported($dataset->datasourceName, $referencedDataset->datasourceName)) {
continue;
}
$referencedOwner = $handler->getDataSourceOwner($referencedDataset->datasourceName);
}
$referencedTableName = $referencedDataset->source;
$referencedColumnName = $referencedDataset->getKeyColumn()->name;
$sql .= ",\n{$indent}CONSTRAINT fk_{$dataset->source}_{$columnName} FOREIGN KEY ({$columnName}) REFERENCES " . (isset($referencedOwner) ? $referencedOwner . '.' : '') . "{$referencedTableName} ({$referencedColumnName})";
}
}
protected function getDataSourceHandler()
{
$environment_metamodel = data_controller_get_environment_metamodel();
$datasource = $environment_metamodel->getDataSource($this->datasourceName);
return DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
}
protected function lookupDataSourceHandler($type) {
return DataSourceQueryFactory::getInstance()->getHandler($type);
}
protected function allowPrimaryKeyCreation(DataSourceHandler $handler, DataControllerCallContext $callcontext, DatasetMetaData $dataset) {
$primaryKeyColumnNames = $dataset->findKeyColumnNames();
if (isset($primaryKeyColumnNames)) {
$environment_metamodel = data_controller_get_environment_metamodel();
$datasource = $environment_metamodel->getDataSource($dataset->datasourceName);
$assembledTableName = assemble_database_entity_name($handler, $datasource->name, $dataset->source);
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
// checking if there are any records with NULL values
$nullValueConditions = array();
foreach ($primaryKeyColumnNames as $columnName) {
$nullValueConditions[] = "$columnName IS NULL";
}
$nullValueSQL = 'SELECT ' . implode(', ', $primaryKeyColumnNames) . " FROM $assembledTableName WHERE " . implode(' OR ', $nullValueConditions);
$datasourceQueryHandler->getExtension('applyPagination')->apply($datasourceQueryHandler, $nullValueSQL, 0, 1);
LogHelper::log_info(new StatementLogMessage('dataset.update.primaryKey.data.check.null', $nullValueSQL));
$nullColumnRecords = $datasourceQueryHandler->executeQuery($callcontext, $datasource, $nullValueSQL);
$isTruncateRequired = isset($nullColumnRecords);
// checking for non-unique values
if (!$isTruncateRequired) {
$nonUniqueValueSQL = 'SELECT ' . implode(', ', $primaryKeyColumnNames)
. " FROM $assembledTableName GROUP BY " . implode(', ', $primaryKeyColumnNames) . ' HAVING COUNT(*) > 1';
$datasourceQueryHandler->getExtension('applyPagination')->apply($datasourceQueryHandler, $nonUniqueValueSQL, 0, 1);
LogHelper::log_info(new StatementLogMessage('dataset.update.primaryKey.data.check', $nonUniqueValueSQL));
$nonUniqueColumnRecords = $datasourceQueryHandler->executeQuery($callcontext, $datasource, $nonUniqueValueSQL);
$isTruncateRequired = isset($nonUniqueColumnRecords);
}
if ($isTruncateRequired) {
$truncateDatasetSQL = $handler->getExtension('truncateTable')->generate($handler, $dataset);
LogHelper::log_info(new StatementLogMessage('dataset.update.primaryKey.data.truncate', $truncateDatasetSQL));
$handler->executeStatement($datasource, $truncateDatasetSQL);
}
}
}
protected function assembleTableName() {
$assembledTableName = $this->name;
$index = strpos($assembledTableName, '.');
if ($index === FALSE) {
$environment_metamodel = data_controller_get_environment_metamodel();
$datasourceName = $this->dataset->datasourceName;
$datasource = $environment_metamodel->getDataSource($datasourceName);
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
$assembledTableName = assemble_database_entity_name($datasourceQueryHandler, $datasource->name, $assembledTableName);
}
return $assembledTableName;
}
protected function assembleTableName()
{
$environment_metamodel = data_controller_get_environment_metamodel();
$datasourceName = $this->dataset->datasourceName;
$datasource = $environment_metamodel->getDataSource($datasourceName);
$datasourceQueryHandler = DataSourceQueryFactory::getInstance()->getHandler($datasource->type);
$owner = $datasourceQueryHandler->getDataSourceOwner($datasourceName);
return (isset($owner) ? $owner . '.' : '') . $this->name;
}