public function loadData($pid, $name, $tmpDir)
{
// Upload to WebDav
$webDav = new WebDav($this->gdClient->getUsername(), $this->gdClient->getPassword(), $this->gdClient->getUserUploadUrl());
$dimensionName = Identifiers::getIdentifier($name);
$tmpFolderNameDimension = "{$pid}-{$dimensionName}-" . uniqid();
$tmpFolderDimension = $tmpDir . '/' . Identifiers::getIdentifier($name);
mkdir($tmpFolderDimension);
$timeDimensionManifest = self::getDataLoadManifest($dimensionName);
file_put_contents("{$tmpFolderDimension}/upload_info.json", $timeDimensionManifest);
copy(__DIR__ . '/time-dimension.csv', "{$tmpFolderDimension}/{$dimensionName}.csv");
$webDav->createFolder($tmpFolderNameDimension);
$webDav->upload("{$tmpFolderDimension}/upload_info.json", $tmpFolderNameDimension);
$webDav->upload("{$tmpFolderDimension}/{$dimensionName}.csv", $tmpFolderNameDimension);
// Run ETL task of time dimensions
try {
$this->gdClient->getDatasets()->loadData($pid, $tmpFolderNameDimension);
} catch (Exception $e) {
$debugFile = "{$tmpFolderDimension}/{$pid}-etl.log";
$logSaved = $webDav->saveLogs($tmpFolderDimension, $debugFile);
if ($logSaved) {
$data = file_get_contents($debugFile);
if ($data) {
$data = json_decode($data, true);
if ($data) {
$e = new Exception(json_decode(file_get_contents($debugFile), true), $e->getCode(), $e);
}
}
}
throw $e;
}
}
public function create($pid, $name, $identifier = null, $template = null)
{
if (!$identifier) {
$identifier = Identifiers::getIdentifier($name);
if (!$identifier) {
throw new Exception("Identifier derived from dimension name '{$name}' is not valid. " . "Choose other name or custom identifier.");
}
}
$call = $this->client->get("/gdc/md/{$pid}/data/sets");
$existingDataSets = [];
foreach ($call['dataSetsInfo']['sets'] as $r) {
$existingDataSets[] = $r['meta']['identifier'];
}
if (!in_array(Identifiers::getDateDimensionId($name, $template), $existingDataSets)) {
$this->client->getDatasets()->executeMaql($pid, sprintf('INCLUDE TEMPLATE "%s" MODIFY (IDENTIFIER "%s", TITLE "%s");', self::getTemplateUrn($template), $identifier, $name));
}
}
public function dropDataSet($pid, $dataSetName)
{
$dataSetId = Identifiers::getIdentifier($dataSetName);
$model = $this->view($pid);
if (isset($model['projectModel']['datasets'])) {
foreach ($model['projectModel']['datasets'] as $i => $dataSet) {
if ($dataSet['dataset']['title'] == $dataSetName) {
unset($model['projectModel']['datasets'][$i]);
break;
}
}
$model['projectModel']['datasets'] = array_values($model['projectModel']['datasets']);
}
$update = $this->diff($pid, $model);
if (count($update['moreDestructiveMaql'])) {
foreach ($update['moreDestructiveMaql'] as $m) {
$this->client->getDatasets()->executeMaql($pid, $m);
}
$this->client->getDatasets()->executeMaql($pid, sprintf('DROP IF EXISTS {dim.%s};', $dataSetId));
$this->client->getDatasets()->executeMaql($pid, sprintf('DROP IF EXISTS {ffld.%s};', $dataSetId));
return $update['description'];
}
return false;
}
public static function getAttributeId($tableName, $attrName)
{
return sprintf('attr.%s.%s', Identifiers::getIdentifier($tableName), Identifiers::getIdentifier($attrName));
}
public static function getDataLoadManifest($identifier, array $columns, $incrementalLoad = false)
{
$manifest = ['dataSetSLIManifest' => ['file' => "{$identifier}.csv", 'dataSet' => $identifier, 'parts' => []]];
foreach ($columns as $columnName => $column) {
if (!isset($column['type'])) {
continue;
}
switch ($column['type']) {
case 'CONNECTION_POINT':
case 'ATTRIBUTE':
if (!isset($column['identifierLabel'])) {
throw Exception::configurationError("Configuration of column {$columnName} is missing 'identifierLabel'");
}
$manifest['dataSetSLIManifest']['parts'][] = ['columnName' => $columnName, 'populates' => [$column['identifierLabel']], 'mode' => $incrementalLoad ? 'INCREMENTAL' : 'FULL', 'referenceKey' => 1];
break;
case 'FACT':
if (!isset($column['identifier'])) {
throw Exception::configurationError("Configuration of column {$columnName} is missing 'identifier'");
}
$manifest['dataSetSLIManifest']['parts'][] = ['columnName' => $columnName, 'populates' => [$column['identifier']], 'mode' => $incrementalLoad ? 'INCREMENTAL' : 'FULL'];
break;
case 'LABEL':
case 'HYPERLINK':
if (!isset($column['identifier'])) {
throw Exception::configurationError("Configuration of column {$columnName} is missing 'identifier'");
}
$manifest['dataSetSLIManifest']['parts'][] = ['columnName' => $columnName, 'populates' => [$column['identifier']], 'mode' => $incrementalLoad ? 'INCREMENTAL' : 'FULL'];
break;
case 'REFERENCE':
if (!isset($column['schemaReferenceConnectionLabel'])) {
throw Exception::configurationError("Configuration of column {$columnName} is missing 'schemaReferenceConnectionLabel'");
}
$manifest['dataSetSLIManifest']['parts'][] = ['columnName' => $columnName, 'populates' => [$column['schemaReferenceConnectionLabel']], 'mode' => $incrementalLoad ? 'INCREMENTAL' : 'FULL', 'referenceKey' => 1];
break;
case 'DATE':
if (!isset($column['identifier'])) {
throw Exception::configurationError("Configuration of column {$columnName} is missing 'identifier'");
}
$manifest['dataSetSLIManifest']['parts'][] = ['columnName' => $columnName, 'populates' => ["{$column['identifier']}.date.mmddyyyy"], 'constraints' => ['date' => (string) $column['format']], 'mode' => $incrementalLoad ? 'INCREMENTAL' : 'FULL', 'referenceKey' => 1];
if (!empty($column['includeTime'])) {
if (!isset($column['identifierTimeFact'])) {
throw Exception::configurationError("Configuration of column {$columnName} is missing 'identifierTimeFact'");
}
$manifest['dataSetSLIManifest']['parts'][] = ['columnName' => $columnName . '_tm', 'populates' => [$column['identifierTimeFact']], 'mode' => $incrementalLoad ? 'INCREMENTAL' : 'FULL'];
$manifest['dataSetSLIManifest']['parts'][] = ['columnName' => $columnName . '_id', 'populates' => ["label.time.second.of.day." . Identifiers::getIdentifier($column['dateDimension'])], 'mode' => $incrementalLoad ? 'INCREMENTAL' : 'FULL', 'referenceKey' => 1];
}
break;
case 'IGNORE':
break;
}
}
return $manifest;
}
