/**
* @param Job $job
* @return array|void
*/
public function execute(Job $job)
{
\Keboola\StorageApi\Config\Reader::$client = $this->storageApi;
$configBucket = \Keboola\StorageApi\Config\Reader::read("sys.c-{$this->appName}", $this->storageApi->token);
$passed = false;
$accountsIds = array();
$accountsOffset = 0;
$accountsLimit = 0;
$jsonParams = $job->getAttribute("params");
if (isset($jsonParams["accountsIds"])) {
$accountsIds = explode(',', $jsonParams["accountsIds"]);
} else {
if (isset($jsonParams["accountsOffset"]) && is_integer($jsonParams["accountsOffset"]) && $jsonParams["accountsOffset"] > 0) {
$accountsOffset = $jsonParams["accountsOffset"];
}
if (isset($jsonParams["accountsLimit"]) && is_integer($jsonParams["accountsLimit"]) && $jsonParams["accountsLimit"] > 0) {
$accountsLimit = $jsonParams["accountsLimit"];
}
}
if (isset($jsonParams["since"])) {
$since = $jsonParams["since"];
} else {
$days = isset($jsonParams["days"]) ? $jsonParams["days"] : 14;
$since = '-' . $days . ' days';
}
$until = isset($jsonParams["until"]) ? $jsonParams["until"] : 'today';
if (strtotime($since) > strtotime($until)) {
return false;
}
if (isset($jsonParams["config"])) {
$jsonParams["configurationId"] = $jsonParams["config"];
}
$reservedTables = array('accounts');
foreach ($configBucket["items"] as $configurationId => $configInstance) {
if (!in_array($configurationId, $reservedTables)) {
if (count($jsonParams) && isset($jsonParams["configurationId"]) && $jsonParams["configurationId"] != $configurationId) {
continue;
}
$passed = true;
$connectionConfig = $configInstance;
unset($connectionConfig["items"]);
$connectionConfig = new \Zend_Config($connectionConfig, true);
$runConfig = $configInstance["items"];
$runConfig = new \Zend_Config($runConfig, true);
try {
$fbImport = new Import($this->appName);
$fbImport->storageApi = $this->storageApi;
$fbImport->runId = $this->storageApi->getRunId();
$fbImport->log("Extraction of row {$configurationId} started");
if (isset($configInstance["paging"])) {
$fbImport->paging = $configInstance["paging"];
}
$fbImport->configurationId = $configurationId;
$fbImport->importConfig = $connectionConfig;
$fbImport->runConfig = $runConfig;
$fbImport->storageApiBucket = "in.c-{$this->appName}-" . $configurationId;
\NDebugger::timer('configuration');
$fbImport->log("Extraction of configuration {$configurationId} started", array('since' => $since, 'until' => $until, 'accountsOffset' => $accountsOffset, 'accountsLimit' => $accountsLimit, 'accountsIds' => $accountsIds));
if (!$this->storageApi->bucketExists($fbImport->storageApiBucket)) {
$this->storageApi->createBucket($this->appName . '-' . $configurationId, \Keboola\StorageApi\Client::STAGE_IN, "Facebook Extractor Data");
}
$tokenInfo = $this->storageApi->getLogData();
$tmpDir = "/tmp/" . $tokenInfo["token"] . "-" . uniqid($configurationId . "-") . "/";
if (!file_exists($tmpDir)) {
mkdir($tmpDir);
}
if (!is_dir($tmpDir)) {
throw new ApplicationException("Temporary directory path ({$tmpDir}) is not a directory", null, null, "TMP_DIR");
}
$fbImport->tmpDir = $tmpDir;
$fbImport->import($since, $until, $accountsOffset, $accountsLimit, $accountsIds);
$duration = \NDebugger::timer('configuration');
$fbImport->log("Extraction of configuration {$configurationId} ended", array(), $duration);
// Cleanup
exec("rm -rf {$tmpDir}");
$fbImport->log("Extraction of row {$configurationId} finished", array(), $duration);
} catch (InvalidTokenException $e) {
throw new UserException("Invalid account {$e->getAccount()} or token for this account: " . $e->getMessage(), $e);
} catch (UserException $e) {
throw $e;
} catch (\Exception $e) {
throw new ApplicationException($e->getMessage(), $e);
}
}
}
if (!$passed) {
throw new UserException("ConfigurationId {$jsonParams["configurationId"]} not found");
}
$response = array("status" => "ok");
return $response;
}
/**
* Run import
* @param string $since
* @param string $until
* @param int $accountsOffset
* @param int $accountsCount
* @param array $accountsIds
* @throws Exception
*/
public function import($since = '-14 days', $until = 'today', $accountsOffset = 0, $accountsCount = 0, $accountsIds = array())
{
$this->_fbApi->runId = $this->runId . '-' . $this->configurationId;
//@TODO DEBUG
$this->_populateSapiTableCache();
$defaultDate = new \DateTime("now", new \DateTimeZone(self::$TIME_ZONE));
$this->defaultEndTime = $defaultDate->format(\DateTime::ATOM);
$this->defaultEndDate = $defaultDate->format('Y-m-d\\T00:00:00P');
$this->defaultInsightsDate = $defaultDate->sub(\DateInterval::createFromDateString("1 days"))->format('Y-m-d\\T00:00:00P');
$this->tmpDir = sprintf('%s/%s-%s', "/tmp/ex-fb", date('Ymd-His'), uniqid());
mkdir($this->tmpDir, 0777, true);
if (!$this->storageApi->tableExists($this->bucket . '.' . self::ACCOUNTS_TABLE_ID)) {
$this->log('Accounts table does not exist in configuration', array(), 0, true);
return;
}
$accountsCsv = $this->storageApi->exportTable($this->bucket . '.' . self::ACCOUNTS_TABLE_ID);
$accounts = \Keboola\StorageApi\Client::parseCsv($accountsCsv);
if (!count($accounts)) {
$this->log('No accounts in configuration table', array(), 0, true);
}
// Create files for each configuration row
$this->_prepareCsvFiles();
// Iterate through each configuration row
foreach ($this->runConfig as $queryNumber => $query) {
$this->currentConfigRowNumber = $queryNumber + 1;
$accountsCounter = 0;
foreach ($accounts as $account) {
if (!isset($account["valid"]) || !$account['valid']) {
continue 1;
}
// Run for the account only if specified or offset and count are matching
if (!(count($accountsIds) && in_array($account['id'], $accountsIds) || !count($accountsIds) && $accountsCounter >= $accountsOffset && ($accountsCount == 0 || $accountsCounter < $accountsOffset + $accountsCount))) {
$accountsCounter++;
continue 1;
}
$accountsCounter++;
if (!isset($account['id']) || !isset($account['token'])) {
$this->log('Wrong configuration of accounts table', array(), 0, true);
continue 1;
}
\NDebugger::timer('account');
$this->_parseQuery($account, $query, $since, $until, $this->_csvFiles[$queryNumber]["handle"]);
}
$this->_uploadCsvFile($queryNumber, $query->table);
$this->log("Extraction of query {$queryNumber} finished", array('queryNumber' => $query), \NDebugger::timer('account'), false);
}
}