/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.'); if (empty($nextUrl)) { return false; } // since validation - cannot be greater than now $now = new \DateTime(); $sinceDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('since')); if ($sinceDateTime && $sinceDateTime > $now) { return false; } $config = $jobConfig->getConfig(); if (!$this->includeParams) { $config['params'] = []; } if (!$this->paramIsQuery) { $config['endpoint'] = $nextUrl; } else { // Create an array from the query string $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray(); $config['params'] = array_replace($config['params'], $responseQuery); } return $client->createRequest($config); }
/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.'); if (empty($nextUrl)) { return false; } // start_time validation // https://developer.zendesk.com/rest_api/docs/core/incremental_export#incremental-ticket-export $now = new \DateTime(); $startDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('start_time')); if ($startDateTime && $startDateTime > $now->modify(sprintf("-%d minutes", self::NEXT_PAGE_FILTER_MINUTES))) { return false; } $config = $jobConfig->getConfig(); if (!$this->includeParams) { $config['params'] = []; } if (!$this->paramIsQuery) { $config['endpoint'] = $nextUrl; } else { // Create an array from the query string $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray(); $config['params'] = array_replace($config['params'], $responseQuery); } return $client->createRequest($config); }
/** * Try to find the data array within $response. * * @param array|object $response * @param array $config * @return array * @todo support array of dataFields * - would return object with results, changing the class' API * - parse would just have to loop through if it returns an object * - and append $type with the dataField * @deprecated Use response module */ public function process($response, JobConfig $jobConfig) { $config = $jobConfig->getConfig(); // If dataField doesn't say where the data is in a response, try to find it! if (!empty($config['dataField'])) { if (is_array($config['dataField'])) { if (empty($config['dataField']['path'])) { throw new UserException("'dataField.path' must be set!"); } $path = $config['dataField']['path']; } elseif (is_scalar($config['dataField'])) { $path = $config['dataField']; } else { throw new UserException("'dataField' must be either a path string or an object with 'path' attribute."); } $data = Utils::getDataFromPath($path, $response, "."); if (empty($data)) { Logger::log('warning', "dataField '{$path}' contains no data!"); $data = []; } elseif (!is_array($data)) { // In case of a single object being returned $data = [$data]; } } elseif (is_array($response)) { // Simplest case, the response is just the dataset $data = $response; } elseif (is_object($response)) { // Find arrays in the response $arrays = []; foreach ($response as $key => $value) { if (is_array($value)) { $arrays[$key] = $value; } // TODO else {$this->metadata[$key] = json_encode($value);} ? return [$data,$metadata]; } $arrayNames = array_keys($arrays); if (count($arrays) == 1) { $data = $arrays[$arrayNames[0]]; } elseif (count($arrays) == 0) { Logger::log('warning', "No data array found in response! (endpoint: {$config['endpoint']})", ['response' => json_encode($response)]); $data = []; } else { $e = new UserException("More than one array found in response! Use 'dataField' parameter to specify a key to the data array. (endpoint: {$config['endpoint']}, arrays in response root: " . join(", ", $arrayNames) . ")"); $e->setData(['response' => json_encode($response), 'arrays found' => $arrayNames]); throw $e; } } else { $e = new UserException('Unknown response from API.'); $e->setData(['response' => json_encode($response)]); throw $e; } return $data; }
/** * @return array */ public function process($response, JobConfig $jobConfig) { if (empty($jobConfig->getConfig()['parseObject'])) { return $response; } $config = $jobConfig->getConfig()['parseObject']; if (!is_object($response)) { if (empty($response)) { return []; } throw new UserException("Data in response is not an object, while one was expected!"); } $path = empty($config['path']) ? "." : $config['path']; $key = empty($config['keyColumn']) ? "rowId" : $config['keyColumn']; return $this->convertObjectWithKeys(Utils::getDataFromPath($path, $response, '.'), $key); }
/** * * Creates key value pairs for `values` property up to 2 leves of nesting * * @param $path * @param $data * @return array * @throws \Keboola\Utils\Exception\NoDataFoundException */ protected function flatten($path, $data) { foreach (Utils::getDataFromPath($path, $data, '.') as $metric) { if ($metric->values && is_array($metric->values)) { $parsedMetrics = []; foreach ($metric->values as $value) { $end_time = ''; if (isset($value->end_time)) { $end_time = $value->end_time; } if (!isset($value->end_time) && $metric->period != 'lifetime') { continue; } // scalar value or empty value if (!isset($value->value) || is_scalar($value->value)) { if (!isset($value->value)) { $val = 0; } else { $val = $value->value; } $parsedMetrics[] = (object) ["id" => $metric->id, "key1" => "", "key2" => "", "end_time" => $end_time, "value" => $val]; continue; } if (is_object($value->value)) { foreach ((array) $value->value as $key1 => $value1) { if (is_object($value1)) { foreach ((array) $value1 as $key2 => $value2) { $parsedMetrics[] = (object) ["id" => $metric->id, "key1" => $key1, "key2" => $key2, "end_time" => $end_time, "value" => $value2]; } } else { $parsedMetrics[] = (object) ["id" => $metric->id, "key1" => $key1, "key2" => "", "end_time" => $end_time, "value" => $value1]; } } continue; } } $metric->values = $parsedMetrics; } $result[] = $metric; } if ($path != '.') { $data->{$path} = $result; } else { $data = $result; } return $data; }
/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { $nextParam = Utils::getDataFromPath($this->responseParam, $response, '.'); if (empty($nextParam)) { return false; } else { $config = $jobConfig->getConfig(); if (!$this->includeParams) { $config['params'] = []; } if (!is_null($this->scrollRequest)) { $config = $this->createScrollRequest($config, $this->scrollRequest); } $config['params'][$this->queryParam] = $nextParam; return $client->createRequest($config); } }
/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.'); if (empty($nextUrl)) { return false; } else { $config = $jobConfig->getConfig(); if (!$this->includeParams) { $config['params'] = []; } if (!$this->paramIsQuery) { $config['endpoint'] = $nextUrl; } else { // Create an array from the query string $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray(); $config['params'] = array_replace($config['params'], $responseQuery); } return $client->createRequest($config); } }
/** * Compare a value from within an object * using the $columnName, $operator and $value * @param \stdClass $object * @return bool * @throws FilterException * @api */ public function compareObject(\stdClass $object) { $value = Utils::getDataFromPath($this->columnName, $object, "."); if (empty($this->multiFilter)) { return $this->compare($value); } else { if ($this->multiOperator == "&") { foreach ($this->multiFilter as $filter) { if (!$filter->compareObject($object)) { return false; } } return true; } elseif ($this->multiOperator == "|") { foreach ($this->multiFilter as $filter) { if ($filter->compareObject($object)) { return true; } } return false; } else { throw new FilterException("MultiFilter is set but MultiOperator is not recognized."); } } }
/** * @param object $response * @return int */ protected function getExpiry(\stdclass $response) { if (!isset($this->auth['expires'])) { return null; } elseif (is_numeric($this->auth['expires'])) { return time() + (int) $this->auth['expires']; } elseif (is_array($this->auth['expires'])) { if (empty($this->auth['expires']['response'])) { throw new UserException("'authentication.expires' must be either an integer or an array with 'response' key containing a path in the response"); } $rExpiry = Utils::getDataFromPath($this->auth['expires']['response'], $response, '.'); $expiry = is_int($rExpiry) ? $rExpiry : strtotime($rExpiry); if (!empty($this->auth['expires']['relative'])) { $expiry += time(); } if ($expiry < time()) { throw new UserException("Login authentication returned expiry time before current time: '{$rExpiry}'"); } return $expiry; } }
/** * @param string $field * @param array $parentResults * @param int $level * @return mixed */ protected function getPlaceholderValue($field, $parentResults, $level, $placeholder) { try { if (!array_key_exists($level, $parentResults)) { $maxLevel = empty($parentResults) ? 0 : max(array_keys($parentResults)) + 1; throw new UserException("Level " . ++$level . " not found in parent results! Maximum level: " . $maxLevel); } return Utils::getDataFromPath($field, $parentResults[$level], ".", false); } catch (NoDataFoundException $e) { throw new UserException("No value found for {$placeholder} in parent result. (level: " . ++$level . ")", 0, null, ['parents' => $parentResults]); } }
/** * Parse the data * @param array|object $data shall be the response body * @param string $type is a WSDL data type (has to be obtained from the WSDL definition) * @param string $path a path to the results list(the array containing each record) within the response * @param string $parent used internally for naming child arrays/columns * @param string $parentId used internally to link child objects to parent */ public function parse($data, $type, $path = null, $parent = null, $parentId = null) { if (!empty($path)) { $data = Utils::getDataFromPath($path, $data); } $fileName = $type; if (empty($this->csvFiles[$fileName])) { $header = array_keys($this->struct[$type]); if ($parentId) { array_push($header, "WSDL_parentId"); } $this->csvFiles[$fileName] = Table::create($fileName, $header, $this->getTemp()); } $handle = $this->csvFiles[$fileName]; $struct = $this->struct[$type]; foreach (Utils::to_assoc($data) as $record) { $row = []; foreach ($struct as $key => $valueType) { if (empty($record[$key])) { $row[$key] = null; } elseif (in_array($valueType, $this->stdTypes)) { $row[$key] = (string) $record[$key]; } elseif (array_key_exists($valueType, $this->struct)) { // Walk through the data type and parse children foreach ($this->struct[$valueType] as $attr => $attrType) { $childId = $type . "_" . $attrType . "_" . (!empty($row["id"]) ? $row["id"] : uniqid()); $row[$key] = $childId; $childPath = "{$key}/{$attr}"; $this->parse($record, $attrType, $childPath, $type, $childId); } } else { $row[$key] = null; } } // FIXME set this in the data before actually caling the fn if ($parentId) { $row["WSDL_parentId"] = $parentId; } $handle->writeRow($row); } }
/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { if (empty($data)) { $this->reset(); return false; } else { $cursor = 0; foreach ($data as $item) { $cursorVal = Utils::getDataFromPath($this->idKey, $item, '.'); if (is_null($this->max) || $cursorVal > $this->max) { $this->max = $cursorVal; } if (is_null($this->min) || $cursorVal < $this->min) { $this->min = $cursorVal; } $cursor = $this->reverse ? $this->min : $this->max; } if (0 !== $this->increment) { if (!is_numeric($cursor)) { throw new UserException("Trying to increment a pointer that is not numeric."); } $cursor += $this->increment; } $jobConfig->setParam($this->param, $cursor); return $client->createRequest($jobConfig->getConfig()); } }