/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.'); if (empty($nextUrl)) { return false; } // since validation - cannot be greater than now $now = new \DateTime(); $sinceDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('since')); if ($sinceDateTime && $sinceDateTime > $now) { return false; } $config = $jobConfig->getConfig(); if (!$this->includeParams) { $config['params'] = []; } if (!$this->paramIsQuery) { $config['endpoint'] = $nextUrl; } else { // Create an array from the query string $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray(); $config['params'] = array_replace($config['params'], $responseQuery); } return $client->createRequest($config); }
/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.'); if (empty($nextUrl)) { return false; } // start_time validation // https://developer.zendesk.com/rest_api/docs/core/incremental_export#incremental-ticket-export $now = new \DateTime(); $startDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('start_time')); if ($startDateTime && $startDateTime > $now->modify(sprintf("-%d minutes", self::NEXT_PAGE_FILTER_MINUTES))) { return false; } $config = $jobConfig->getConfig(); if (!$this->includeParams) { $config['params'] = []; } if (!$this->paramIsQuery) { $config['endpoint'] = $nextUrl; } else { // Create an array from the query string $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray(); $config['params'] = array_replace($config['params'], $responseQuery); } return $client->createRequest($config); }
/** * @return array */ public function process($response, JobConfig $jobConfig) { if (empty($jobConfig->getConfig()['parseObject'])) { return $response; } $config = $jobConfig->getConfig()['parseObject']; if (!is_object($response)) { if (empty($response)) { return []; } throw new UserException("Data in response is not an object, while one was expected!"); } $path = empty($config['path']) ? "." : $config['path']; $key = empty($config['keyColumn']) ? "rowId" : $config['keyColumn']; return $this->convertObjectWithKeys(Utils::getDataFromPath($path, $response, '.'), $key); }
/** * @param JobConfig $jobConfig * @return ScrollerInterface */ public function getScrollerForJob(JobConfig $jobConfig) { if (empty($jobConfig->getConfig()['scroller'])) { if (empty($this->defaultScroller)) { return new NoScroller(); } if (!array_key_exists($this->defaultScroller, $this->scrollers)) { throw new UserException("Default scroller '{$this->defaultScroller}' does not exist"); } return $this->scrollers[$this->defaultScroller]; } $scrollerId = $jobConfig->getConfig()['scroller']; if (empty($this->scrollers[$scrollerId])) { throw new UserException("Scroller '{$scrollerId}' not set in API definitions. Scrollers defined: " . join(', ', array_keys($this->scrollers))); } return $this->scrollers[$scrollerId]; }
/** * Try to find the data array within $response. * * @param array|object $response * @param array $config * @return array * @todo support array of dataFields * - would return object with results, changing the class' API * - parse would just have to loop through if it returns an object * - and append $type with the dataField * @deprecated Use response module */ public function process($response, JobConfig $jobConfig) { $config = $jobConfig->getConfig(); // If dataField doesn't say where the data is in a response, try to find it! if (!empty($config['dataField'])) { if (is_array($config['dataField'])) { if (empty($config['dataField']['path'])) { throw new UserException("'dataField.path' must be set!"); } $path = $config['dataField']['path']; } elseif (is_scalar($config['dataField'])) { $path = $config['dataField']; } else { throw new UserException("'dataField' must be either a path string or an object with 'path' attribute."); } $data = Utils::getDataFromPath($path, $response, "."); if (empty($data)) { Logger::log('warning', "dataField '{$path}' contains no data!"); $data = []; } elseif (!is_array($data)) { // In case of a single object being returned $data = [$data]; } } elseif (is_array($response)) { // Simplest case, the response is just the dataset $data = $response; } elseif (is_object($response)) { // Find arrays in the response $arrays = []; foreach ($response as $key => $value) { if (is_array($value)) { $arrays[$key] = $value; } // TODO else {$this->metadata[$key] = json_encode($value);} ? return [$data,$metadata]; } $arrayNames = array_keys($arrays); if (count($arrays) == 1) { $data = $arrays[$arrayNames[0]]; } elseif (count($arrays) == 0) { Logger::log('warning', "No data array found in response! (endpoint: {$config['endpoint']})", ['response' => json_encode($response)]); $data = []; } else { $e = new UserException("More than one array found in response! Use 'dataField' parameter to specify a key to the data array. (endpoint: {$config['endpoint']}, arrays in response root: " . join(", ", $arrayNames) . ")"); $e->setData(['response' => json_encode($response), 'arrays found' => $arrayNames]); throw $e; } } else { $e = new UserException('Unknown response from API.'); $e->setData(['response' => json_encode($response)]); throw $e; } return $data; }
/** * @param $response * @param JobConfig $jobConfig * @return array */ public function process($response, JobConfig $jobConfig) { $config = $jobConfig->getConfig(); if (!isset($config['parser']['method'])) { return $response; } $path = empty($config['dataField']) ? "." : $config['dataField']; if ($config['parser']['method'] == 'facebook.insights') { return $this->flatten($path, $response); } return $response; }
/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { $nextParam = Utils::getDataFromPath($this->responseParam, $response, '.'); if (empty($nextParam)) { return false; } else { $config = $jobConfig->getConfig(); if (!$this->includeParams) { $config['params'] = []; } if (!is_null($this->scrollRequest)) { $config = $this->createScrollRequest($config, $this->scrollRequest); } $config['params'][$this->queryParam] = $nextParam; return $client->createRequest($config); } }
/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { $nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.'); if (empty($nextUrl)) { return false; } else { $config = $jobConfig->getConfig(); if (!$this->includeParams) { $config['params'] = []; } if (!$this->paramIsQuery) { $config['endpoint'] = $nextUrl; } else { // Create an array from the query string $responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray(); $config['params'] = array_replace($config['params'], $responseQuery); } return $client->createRequest($config); } }
/** * {@inheritdoc} */ public function getFirstRequest(ClientInterface $client, JobConfig $jobConfig) { return $client->createRequest($jobConfig->getConfig()); }
/** * Create a child job with current client and parser * @param JobConfig $config * @return static */ protected function createChild(JobConfig $config, array $parentResults) { // Clone the config to prevent overwriting the placeholder(s) in endpoint $job = new static(clone $config, $this->client, $this->parser); $params = []; $placeholders = !empty($config->getConfig()['placeholders']) ? $config->getConfig()['placeholders'] : []; if (empty($placeholders)) { Logger::log("WARNING", "No 'placeholders' set for '" . $config->getConfig()['endpoint'] . "'"); } foreach ($placeholders as $placeholder => $field) { $params[$placeholder] = $this->getPlaceholder($placeholder, $field, $parentResults); } // Add parent params as well (for 'tagging' child-parent data) // Same placeholder in deeper nesting replaces parent value if (!empty($this->parentParams)) { $params = array_replace($this->parentParams, $params); } $job->setParams($params); $job->setParentResults($parentResults); return $job; }
/** * {@inheritdoc} */ public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data) { if (empty($data)) { $this->reset(); return false; } else { $cursor = 0; foreach ($data as $item) { $cursorVal = Utils::getDataFromPath($this->idKey, $item, '.'); if (is_null($this->max) || $cursorVal > $this->max) { $this->max = $cursorVal; } if (is_null($this->min) || $cursorVal < $this->min) { $this->min = $cursorVal; } $cursor = $this->reverse ? $this->min : $this->max; } if (0 !== $this->increment) { if (!is_numeric($cursor)) { throw new UserException("Trying to increment a pointer that is not numeric."); } $cursor += $this->increment; } $jobConfig->setParam($this->param, $cursor); return $client->createRequest($jobConfig->getConfig()); } }
/** * Returns a config with scroller params * @param JobConfig $jobConfig * @return array */ protected function getParams(JobConfig $jobConfig) { $config = $jobConfig->getConfig(); $scrollParams = [$this->limitParam => $this->getLimit($jobConfig), $this->offsetParam => $this->pointer]; $config['params'] = array_replace($jobConfig->getParams(), $scrollParams); return $config; }
/** * @param JobConfig $config */ public static function create(JobConfig $config) { $filters = empty($config->getConfig()['responseFilter']) ? [] : (is_array($config->getConfig()['responseFilter']) ? $config->getConfig()['responseFilter'] : [$config->getConfig()['responseFilter']]); $delimiter = empty($config->getConfig()['responseFilterDelimiter']) ? self::DEFAULT_DELIMITER : $config->getConfig()['responseFilterDelimiter']; return new self($filters, $delimiter); }
/** * @return string */ protected function getDataType() { $config = $this->config->getConfig(); $type = !empty($config['dataType']) ? $config['dataType'] : $config['endpoint']; return $type; }