/** * Perform the actual download. * * @param string $url * @param int $delay * @param bool $opendata * * @return array */ private function doDownload($url, $delay = 5, $opendata = false) { $client = PJClient::getInstance(); if ($this->proxyManager->useProxy()) { $client->addOption('--proxy=' . $this->proxyManager->getProxyAddress()); } $client->addOption('--load-images=false'); $request = $client->getMessageFactory()->createRequest($this->fullURL($url, true, $opendata)); $request->setDelay($delay); $request->setTimeout(60000); $request->addHeader('User-Agent', $this->identity->getUserAgent($opendata)); $response = $client->getMessageFactory()->createResponse(); $client->send($request, $response); $status = $response->getStatus(); $html = $response->getContent(); sleep(max(10, min(0, $this->proxyManager->getProxyLastUsageTime() + 10000 - time()))); return ['status' => $status, 'html' => preg_replace('|charset="?windows-1251"?|u', 'charset="utf-8"', $html)]; }