/**
  * {@inheritdoc}
  */
 public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
 {
     $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
     if (empty($nextUrl)) {
         return false;
     }
     // since validation - cannot be greater than now
     $now = new \DateTime();
     $sinceDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('since'));
     if ($sinceDateTime && $sinceDateTime > $now) {
         return false;
     }
     $config = $jobConfig->getConfig();
     if (!$this->includeParams) {
         $config['params'] = [];
     }
     if (!$this->paramIsQuery) {
         $config['endpoint'] = $nextUrl;
     } else {
         // Create an array from the query string
         $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
         $config['params'] = array_replace($config['params'], $responseQuery);
     }
     return $client->createRequest($config);
 }
 /**
  * {@inheritdoc}
  */
 public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
 {
     $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
     if (empty($nextUrl)) {
         return false;
     }
     // start_time validation
     // https://developer.zendesk.com/rest_api/docs/core/incremental_export#incremental-ticket-export
     $now = new \DateTime();
     $startDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('start_time'));
     if ($startDateTime && $startDateTime > $now->modify(sprintf("-%d minutes", self::NEXT_PAGE_FILTER_MINUTES))) {
         return false;
     }
     $config = $jobConfig->getConfig();
     if (!$this->includeParams) {
         $config['params'] = [];
     }
     if (!$this->paramIsQuery) {
         $config['endpoint'] = $nextUrl;
     } else {
         // Create an array from the query string
         $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
         $config['params'] = array_replace($config['params'], $responseQuery);
     }
     return $client->createRequest($config);
 }
 protected function getFileUrl($response, $attempt = 0)
 {
     Logger::log("debug", "Retrieving report file URL (attempt = {$attempt}).", Utils::to_assoc($response));
     switch ($response->status) {
         case "REPORT_AVAILABLE":
             Logger::log("debug", "Report {$this->config["reportId"]} is ready! URL: {$response->urls->apiUrl}", Utils::to_assoc($response));
             if ($response->format != "CSV") {
                 throw new UserException("[415] The report {$this->config["reportId"]} export format is not CSV.");
             }
             $result = $response->urls->apiUrl;
             break;
         case "PROCESSING":
             if ($attempt > 10) {
                 throw new UserException("The report export timed out.");
             }
             $attempt++;
             $sleep = pow(2, $attempt);
             Logger::log("debug", "Report {$this->config["reportId"]} is still not ready (status = {$response->status}). Waiting for {$sleep}s.");
             sleep($sleep);
             $result = $this->getFileUrl($this->download($this->client->createRequest("GET", "userprofiles/{$this->config["profileId"]}/reports/{$this->config["reportId"]}/files/{$response->id}")), $attempt);
             break;
         default:
             throw new SyrupComponentException(500, "Unknown report status {$response->status}");
     }
     return $result;
 }
 /**
  * Try to find the data array within $response.
  *
  * @param array|object $response
  * @param array $config
  * @return array
  * @todo support array of dataFields
  *     - would return object with results, changing the class' API
  *     - parse would just have to loop through if it returns an object
  *     - and append $type with the dataField
  * @deprecated Use response module
  */
 public function process($response, JobConfig $jobConfig)
 {
     $config = $jobConfig->getConfig();
     // If dataField doesn't say where the data is in a response, try to find it!
     if (!empty($config['dataField'])) {
         if (is_array($config['dataField'])) {
             if (empty($config['dataField']['path'])) {
                 throw new UserException("'dataField.path' must be set!");
             }
             $path = $config['dataField']['path'];
         } elseif (is_scalar($config['dataField'])) {
             $path = $config['dataField'];
         } else {
             throw new UserException("'dataField' must be either a path string or an object with 'path' attribute.");
         }
         $data = Utils::getDataFromPath($path, $response, ".");
         if (empty($data)) {
             Logger::log('warning', "dataField '{$path}' contains no data!");
             $data = [];
         } elseif (!is_array($data)) {
             // In case of a single object being returned
             $data = [$data];
         }
     } elseif (is_array($response)) {
         // Simplest case, the response is just the dataset
         $data = $response;
     } elseif (is_object($response)) {
         // Find arrays in the response
         $arrays = [];
         foreach ($response as $key => $value) {
             if (is_array($value)) {
                 $arrays[$key] = $value;
             }
             // TODO else {$this->metadata[$key] = json_encode($value);} ? return [$data,$metadata];
         }
         $arrayNames = array_keys($arrays);
         if (count($arrays) == 1) {
             $data = $arrays[$arrayNames[0]];
         } elseif (count($arrays) == 0) {
             Logger::log('warning', "No data array found in response! (endpoint: {$config['endpoint']})", ['response' => json_encode($response)]);
             $data = [];
         } else {
             $e = new UserException("More than one array found in response! Use 'dataField' parameter to specify a key to the data array. (endpoint: {$config['endpoint']}, arrays in response root: " . join(", ", $arrayNames) . ")");
             $e->setData(['response' => json_encode($response), 'arrays found' => $arrayNames]);
             throw $e;
         }
     } else {
         $e = new UserException('Unknown response from API.');
         $e->setData(['response' => json_encode($response)]);
         throw $e;
     }
     return $data;
 }
Esempio n. 5
0
 /**
  * @param array|\stdClass $functions
  * @param array $params ['attr' => $attributesArray, ...]
  * @param Builder $builder
  * @return array
  */
 public static function build($functions, array $params = [], Builder $builder = null)
 {
     if (is_null($builder)) {
         $builder = new Builder();
     }
     $functions = (array) Utils::arrayToObject($functions);
     try {
         array_walk($functions, function (&$value, $key) use($params, $builder) {
             $value = !is_object($value) ? $value : $builder->run($value, $params);
         });
     } catch (UserScriptException $e) {
         throw new UserException('User script error: ' . $e->getMessage());
     }
     return $functions;
 }
 public function __construct(ScrollerInterface $scroller, array $config)
 {
     if (!empty($config['forceStop'])) {
         if (!empty($config['forceStop']['pages'])) {
             $this->pageLimit = $config['forceStop']['pages'];
         }
         if (!empty($config['forceStop']['time'])) {
             $this->timeLimit = is_int($config['forceStop']['time']) ? $config['forceStop']['time'] : strtotime($config['forceStop']['time'], 0);
         }
         if (!empty($config['forceStop']['volume'])) {
             $this->volumeLimit = Utils::return_bytes($config['forceStop']['volume']);
         }
     }
     parent::__construct($scroller, $config);
     $this->reset();
 }
 /**
  * @return array
  */
 public function process($response, JobConfig $jobConfig)
 {
     if (empty($jobConfig->getConfig()['parseObject'])) {
         return $response;
     }
     $config = $jobConfig->getConfig()['parseObject'];
     if (!is_object($response)) {
         if (empty($response)) {
             return [];
         }
         throw new UserException("Data in response is not an object, while one was expected!");
     }
     $path = empty($config['path']) ? "." : $config['path'];
     $key = empty($config['keyColumn']) ? "rowId" : $config['keyColumn'];
     return $this->convertObjectWithKeys(Utils::getDataFromPath($path, $response, '.'), $key);
 }
 /**
  *
  * Creates key value pairs for `values` property up to 2 leves of nesting
  *
  * @param $path
  * @param $data
  * @return array
  * @throws \Keboola\Utils\Exception\NoDataFoundException
  */
 protected function flatten($path, $data)
 {
     foreach (Utils::getDataFromPath($path, $data, '.') as $metric) {
         if ($metric->values && is_array($metric->values)) {
             $parsedMetrics = [];
             foreach ($metric->values as $value) {
                 $end_time = '';
                 if (isset($value->end_time)) {
                     $end_time = $value->end_time;
                 }
                 if (!isset($value->end_time) && $metric->period != 'lifetime') {
                     continue;
                 }
                 // scalar value or empty value
                 if (!isset($value->value) || is_scalar($value->value)) {
                     if (!isset($value->value)) {
                         $val = 0;
                     } else {
                         $val = $value->value;
                     }
                     $parsedMetrics[] = (object) ["id" => $metric->id, "key1" => "", "key2" => "", "end_time" => $end_time, "value" => $val];
                     continue;
                 }
                 if (is_object($value->value)) {
                     foreach ((array) $value->value as $key1 => $value1) {
                         if (is_object($value1)) {
                             foreach ((array) $value1 as $key2 => $value2) {
                                 $parsedMetrics[] = (object) ["id" => $metric->id, "key1" => $key1, "key2" => $key2, "end_time" => $end_time, "value" => $value2];
                             }
                         } else {
                             $parsedMetrics[] = (object) ["id" => $metric->id, "key1" => $key1, "key2" => "", "end_time" => $end_time, "value" => $value1];
                         }
                     }
                     continue;
                 }
             }
             $metric->values = $parsedMetrics;
         }
         $result[] = $metric;
     }
     if ($path != '.') {
         $data->{$path} = $result;
     } else {
         $data = $result;
     }
     return $data;
 }
Esempio n. 9
0
 /**
  * {@inheritdoc}
  */
 public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
 {
     $nextParam = Utils::getDataFromPath($this->responseParam, $response, '.');
     if (empty($nextParam)) {
         return false;
     } else {
         $config = $jobConfig->getConfig();
         if (!$this->includeParams) {
             $config['params'] = [];
         }
         if (!is_null($this->scrollRequest)) {
             $config = $this->createScrollRequest($config, $this->scrollRequest);
         }
         $config['params'][$this->queryParam] = $nextParam;
         return $client->createRequest($config);
     }
 }
Esempio n. 10
0
 public function __construct(array $authorization)
 {
     if (empty($authorization['oauth_api']['credentials'])) {
         throw new UserException("OAuth API credentials not supplied in config");
     }
     $oauthApiDetails = $authorization['oauth_api']['credentials'];
     foreach (['#data', 'appKey', '#appSecret'] as $key) {
         if (empty($oauthApiDetails[$key])) {
             throw new UserException("Missing '{$key}' for OAuth 1.0 authorization");
         }
     }
     $data = Utils::json_decode($oauthApiDetails['#data']);
     $this->token = $data->oauth_token;
     $this->tokenSecret = $data->oauth_token_secret;
     $this->consumerKey = $oauthApiDetails['appKey'];
     $this->consumerSecret = $oauthApiDetails['#appSecret'];
 }
Esempio n. 11
0
 /**
  * @param RestClient $client
  */
 public function authenticateClient(RestClient $client)
 {
     $sub = new UrlSignature();
     // Create array of objects instead of arrays from YML
     $q = (array) Utils::arrayToObject($this->query);
     $sub->setSignatureGenerator(function (array $requestInfo = []) use($q) {
         $params = array_merge($requestInfo, ['attr' => $this->attrs]);
         $query = [];
         try {
             foreach ($q as $key => $value) {
                 $query[$key] = is_scalar($value) ? $value : $this->builder->run($value, $params);
             }
         } catch (UserScriptException $e) {
             throw new UserException("Error in query authentication script: " . $e->getMessage());
         }
         return $query;
     });
     $client->getClient()->getEmitter()->attach($sub);
 }
Esempio n. 12
0
 public function __construct($authorization, array $api)
 {
     if (empty($authorization['oauth_api']['credentials'])) {
         throw new UserException("OAuth API credentials not supplied in config");
     }
     $oauthApiDetails = $authorization['oauth_api']['credentials'];
     foreach (['#data', 'appKey', '#appSecret'] as $key) {
         if (empty($oauthApiDetails[$key])) {
             throw new UserException("Missing '{$key}' for OAuth 2.0 authorization");
         }
     }
     try {
         $oAuthData = Utils::json_decode($oauthApiDetails['#data'], true);
     } catch (JsonDecodeException $e) {
         throw new UserException("The OAuth data is not a valid JSON");
     }
     $consumerData = ['client_id' => $oauthApiDetails['appKey'], 'client_secret' => $oauthApiDetails['#appSecret']];
     $this->params = ['consumer' => $consumerData, 'user' => $oAuthData];
     $this->auth = $api['authentication'];
 }
Esempio n. 13
0
 /**
  * {@inheritdoc}
  */
 public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
 {
     $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
     if (empty($nextUrl)) {
         return false;
     } else {
         $config = $jobConfig->getConfig();
         if (!$this->includeParams) {
             $config['params'] = [];
         }
         if (!$this->paramIsQuery) {
             $config['endpoint'] = $nextUrl;
         } else {
             // Create an array from the query string
             $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
             $config['params'] = array_replace($config['params'], $responseQuery);
         }
         return $client->createRequest($config);
     }
 }
Esempio n. 14
0
 /**
  * @param string|int $limit
  */
 public function setMemoryLimit($limit)
 {
     $this->memoryLimit = Utils::return_bytes($limit);
 }
Esempio n. 15
0
 /**
  * @param array|object $definitions
  */
 protected function addGenerator($subscriber, $definitions, $authorization)
 {
     // Create array of objects instead of arrays from YML
     $q = (array) Utils::arrayToObject($definitions);
     $subscriber->setSignatureGenerator(function (array $requestInfo = []) use($q, $authorization) {
         $params = array_merge($requestInfo, ['authorization' => $authorization]);
         $result = [];
         try {
             foreach ($q as $key => $value) {
                 $result[$key] = is_scalar($value) ? $value : $this->builder->run($value, $params);
             }
         } catch (UserScriptException $e) {
             throw new UserException("Error in OAuth authentication script: " . $e->getMessage());
         }
         return $result;
     });
 }
Esempio n. 16
0
 /**
  * Analyze row of input data & create $this->struct
  *
  * @param mixed $row
  * @param string $type
  * @return void
  */
 protected function analyzeRow($row, $type)
 {
     // Current row's structure
     $struct = [];
     $rowType = $this->getType($row);
     // If the row is scalar, make it a {"data" => $value} object
     if (is_scalar($row)) {
         $struct[Parser::DATA_COLUMN] = $this->getType($row);
     } elseif (is_object($row)) {
         // process each property of the object
         foreach ($row as $key => $field) {
             $fieldType = $this->getType($field);
             if ($fieldType == "object") {
                 // Only assign the type if the object isn't empty
                 if (Utils::isEmptyObject($field)) {
                     continue;
                 }
                 $this->analyzeRow($field, $type . "." . $key);
             } elseif ($fieldType == "array") {
                 $arrayType = $this->analyze($field, $type . "." . $key);
                 if (false !== $arrayType) {
                     $fieldType = 'arrayOf' . $arrayType;
                 } else {
                     $fieldType = 'NULL';
                 }
             }
             $struct[$key] = $fieldType;
         }
     } elseif ($this->nestedArrayAsJson && is_array($row)) {
         $this->log->log("WARNING", "Unsupported array nesting in '{$type}'! Converting to JSON string.", ['row' => $row]);
         $rowType = $struct[Parser::DATA_COLUMN] = 'string';
     } elseif (is_null($row)) {
         // do nothing
     } else {
         throw new JsonParserException("Unsupported data row in '{$type}'!", ['row' => $row]);
     }
     $this->getStruct()->add($type, $struct);
     return $rowType;
 }
Esempio n. 17
0
 /**
  * Compare a value from within an object
  * using the $columnName, $operator and $value
  * @param \stdClass $object
  * @return bool
  * @throws FilterException
  * @api
  */
 public function compareObject(\stdClass $object)
 {
     $value = Utils::getDataFromPath($this->columnName, $object, ".");
     if (empty($this->multiFilter)) {
         return $this->compare($value);
     } else {
         if ($this->multiOperator == "&") {
             foreach ($this->multiFilter as $filter) {
                 if (!$filter->compareObject($object)) {
                     return false;
                 }
             }
             return true;
         } elseif ($this->multiOperator == "|") {
             foreach ($this->multiFilter as $filter) {
                 if ($filter->compareObject($object)) {
                     return true;
                 }
             }
             return false;
         } else {
             throw new FilterException("MultiFilter is set but MultiOperator is not recognized.");
         }
     }
 }
Esempio n. 18
0
 /**
  * @param object $response
  * @return int
  */
 protected function getExpiry(\stdclass $response)
 {
     if (!isset($this->auth['expires'])) {
         return null;
     } elseif (is_numeric($this->auth['expires'])) {
         return time() + (int) $this->auth['expires'];
     } elseif (is_array($this->auth['expires'])) {
         if (empty($this->auth['expires']['response'])) {
             throw new UserException("'authentication.expires' must be either an integer or an array with 'response' key containing a path in the response");
         }
         $rExpiry = Utils::getDataFromPath($this->auth['expires']['response'], $response, '.');
         $expiry = is_int($rExpiry) ? $rExpiry : strtotime($rExpiry);
         if (!empty($this->auth['expires']['relative'])) {
             $expiry += time();
         }
         if ($expiry < time()) {
             throw new UserException("Login authentication returned expiry time before current time: '{$rExpiry}'");
         }
         return $expiry;
     }
 }
Esempio n. 19
0
 /**
  * @param string $field
  * @param array $parentResults
  * @param int $level
  * @return mixed
  */
 protected function getPlaceholderValue($field, $parentResults, $level, $placeholder)
 {
     try {
         if (!array_key_exists($level, $parentResults)) {
             $maxLevel = empty($parentResults) ? 0 : max(array_keys($parentResults)) + 1;
             throw new UserException("Level " . ++$level . " not found in parent results! Maximum level: " . $maxLevel);
         }
         return Utils::getDataFromPath($field, $parentResults[$level], ".", false);
     } catch (NoDataFoundException $e) {
         throw new UserException("No value found for {$placeholder} in parent result. (level: " . ++$level . ")", 0, null, ['parents' => $parentResults]);
     }
 }
Esempio n. 20
0
 /**
  * {@inheritdoc}
  */
 public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
 {
     if (empty($data)) {
         $this->reset();
         return false;
     } else {
         $cursor = 0;
         foreach ($data as $item) {
             $cursorVal = Utils::getDataFromPath($this->idKey, $item, '.');
             if (is_null($this->max) || $cursorVal > $this->max) {
                 $this->max = $cursorVal;
             }
             if (is_null($this->min) || $cursorVal < $this->min) {
                 $this->min = $cursorVal;
             }
             $cursor = $this->reverse ? $this->min : $this->max;
         }
         if (0 !== $this->increment) {
             if (!is_numeric($cursor)) {
                 throw new UserException("Trying to increment a pointer that is not numeric.");
             }
             $cursor += $this->increment;
         }
         $jobConfig->setParam($this->param, $cursor);
         return $client->createRequest($jobConfig->getConfig());
     }
 }
Esempio n. 21
0
 protected static function getRetryDelay($retries, AbstractTransferEvent $event, $headerName)
 {
     if (is_null($event->getResponse()) || !$event->getResponse()->hasHeader($headerName)) {
         return RetrySubscriber::exponentialDelay($retries, $event);
     }
     $retryAfter = $event->getResponse()->getHeader($headerName);
     if (is_numeric($retryAfter)) {
         if ($retryAfter < time() - strtotime('1 day', 0)) {
             return $retryAfter;
         } else {
             return $retryAfter - time();
         }
     }
     if (Utils::isValidDateTimeString($retryAfter, DATE_RFC1123)) {
         $date = \DateTime::createFromFormat(DATE_RFC1123, $retryAfter);
         return $date->getTimestamp() - time();
     }
     return RetrySubscriber::exponentialDelay($retries, $event);
 }
 /**
  * @expectedException \Keboola\Juicer\Exception\UserException
  * @expectedExceptionMessage User script error: date() expects at least 1 parameter, 0 given
  */
 public function testBuildParamsException()
 {
     $cfg = new JobConfig(1, ['params' => Utils::json_decode('{
             "filters": {
                 "function": "date"
             }
         }')]);
     $job = $this->getJob($cfg);
     $job->setAttributes(['das.attribute' => "something interesting"]);
     $job->setMetadata(['time' => ['previousStart' => 0, 'currentStart' => time()]]);
     $job->setBuilder(new Builder());
     self::callMethod($job, 'buildParams', [$cfg]);
 }
Esempio n. 23
0
 /**
  * Parse the data
  * @param array|object $data shall be the response body
  * @param string $type is a WSDL data type (has to be obtained from the WSDL definition)
  * @param string $path a path to the results list(the array containing each record) within the response
  * @param string $parent used internally for naming child arrays/columns
  * @param string $parentId used internally to link child objects to parent
  */
 public function parse($data, $type, $path = null, $parent = null, $parentId = null)
 {
     if (!empty($path)) {
         $data = Utils::getDataFromPath($path, $data);
     }
     $fileName = $type;
     if (empty($this->csvFiles[$fileName])) {
         $header = array_keys($this->struct[$type]);
         if ($parentId) {
             array_push($header, "WSDL_parentId");
         }
         $this->csvFiles[$fileName] = Table::create($fileName, $header, $this->getTemp());
     }
     $handle = $this->csvFiles[$fileName];
     $struct = $this->struct[$type];
     foreach (Utils::to_assoc($data) as $record) {
         $row = [];
         foreach ($struct as $key => $valueType) {
             if (empty($record[$key])) {
                 $row[$key] = null;
             } elseif (in_array($valueType, $this->stdTypes)) {
                 $row[$key] = (string) $record[$key];
             } elseif (array_key_exists($valueType, $this->struct)) {
                 // Walk through the data type and parse children
                 foreach ($this->struct[$valueType] as $attr => $attrType) {
                     $childId = $type . "_" . $attrType . "_" . (!empty($row["id"]) ? $row["id"] : uniqid());
                     $row[$key] = $childId;
                     $childPath = "{$key}/{$attr}";
                     $this->parse($record, $attrType, $childPath, $type, $childId);
                 }
             } else {
                 $row[$key] = null;
             }
         }
         // FIXME set this in the data before actually caling the fn
         if ($parentId) {
             $row["WSDL_parentId"] = $parentId;
         }
         $handle->writeRow($row);
     }
 }
Esempio n. 24
0
 public function testNoStrictScalarChange()
 {
     $parser = $this->getParser();
     $data = Utils::json_decode('[
         {"field": 128},
         {"field": "string"},
         {"field": true}
     ]');
     $parser->process($data, 'threepack');
     self::assertEquals(['"field"' . PHP_EOL, '"128"' . PHP_EOL, '"string"' . PHP_EOL, '"1"' . PHP_EOL], file($parser->getCsvFiles()['threepack']->getPathname()));
 }
Esempio n. 25
0
 protected function loadJson($fileName)
 {
     $testFilesPath = $this->getDataDir() . $fileName . ".json";
     $file = file_get_contents($testFilesPath);
     return Utils::json_decode($file);
 }