/**
* Execute console command.
*/
public function handle()
{
if ($this->option('single')) {
$output = [];
exec('pgrep -l -f "^php (.*?)artisan start"', $output);
foreach ($output as $line) {
$pid = preg_replace('|([0-9]+)(\\s.*)|u', '$1', $line);
if ($pid != getmypid()) {
exec("kill -9 {$pid}");
}
}
}
if ($this->option('proxies')) {
$this->workers = $this->option('proxies');
$this->proxyManager->useProxy(true);
}
if ($this->proxyManager->useProxy()) {
$this->proxyManager->connect($this->workers, $this->option('kill_old_proxies'));
}
if ($job_id = $this->option('job')) {
$job = Job::find($job_id);
if ($job) {
$job->execute();
} else {
_log("Job {$job_id} is not found.");
}
return;
}
if (!$this->jobsManager->count()) {
_log('No jobs found. Initializing a new discovery and download jobs.');
$this->discoverer->discoverNewLaws();
$this->downloader->downloadNewLaws();
}
$this->jobsManager->launch($this->workers);
}
/**
* Perform the actual download.
*
* @param string $url
* @param int $delay
* @param bool $opendata
*
* @return array
*/
private function doDownload($url, $delay = 5, $opendata = false)
{
$client = PJClient::getInstance();
if ($this->proxyManager->useProxy()) {
$client->addOption('--proxy=' . $this->proxyManager->getProxyAddress());
}
$client->addOption('--load-images=false');
$request = $client->getMessageFactory()->createRequest($this->fullURL($url, true, $opendata));
$request->setDelay($delay);
$request->setTimeout(60000);
$request->addHeader('User-Agent', $this->identity->getUserAgent($opendata));
$response = $client->getMessageFactory()->createResponse();
$client->send($request, $response);
$status = $response->getStatus();
$html = $response->getContent();
sleep(max(10, min(0, $this->proxyManager->getProxyLastUsageTime() + 10000 - time())));
return ['status' => $status, 'html' => preg_replace('|charset="?windows-1251"?|u', 'charset="utf-8"', $html)];
}
