Example #1
0
 /**
  * Execute console command.
  */
 public function handle()
 {
     if ($this->option('single')) {
         $output = [];
         exec('pgrep -l -f "^php (.*?)artisan start"', $output);
         foreach ($output as $line) {
             $pid = preg_replace('|([0-9]+)(\\s.*)|u', '$1', $line);
             if ($pid != getmypid()) {
                 exec("kill -9 {$pid}");
             }
         }
     }
     if ($this->option('proxies')) {
         $this->workers = $this->option('proxies');
         $this->proxyManager->useProxy(true);
     }
     if ($this->proxyManager->useProxy()) {
         $this->proxyManager->connect($this->workers, $this->option('kill_old_proxies'));
     }
     if ($job_id = $this->option('job')) {
         $job = Job::find($job_id);
         if ($job) {
             $job->execute();
         } else {
             _log("Job {$job_id} is not found.");
         }
         return;
     }
     if (!$this->jobsManager->count()) {
         _log('No jobs found. Initializing a new discovery and download jobs.');
         $this->discoverer->discoverNewLaws();
         $this->downloader->downloadNewLaws();
     }
     $this->jobsManager->launch($this->workers);
 }
Example #2
0
 /**
  * Perform the actual download.
  *
  * @param string $url
  * @param int $delay
  * @param bool $opendata
  *
  * @return array
  */
 private function doDownload($url, $delay = 5, $opendata = false)
 {
     $client = PJClient::getInstance();
     if ($this->proxyManager->useProxy()) {
         $client->addOption('--proxy=' . $this->proxyManager->getProxyAddress());
     }
     $client->addOption('--load-images=false');
     $request = $client->getMessageFactory()->createRequest($this->fullURL($url, true, $opendata));
     $request->setDelay($delay);
     $request->setTimeout(60000);
     $request->addHeader('User-Agent', $this->identity->getUserAgent($opendata));
     $response = $client->getMessageFactory()->createResponse();
     $client->send($request, $response);
     $status = $response->getStatus();
     $html = $response->getContent();
     sleep(max(10, min(0, $this->proxyManager->getProxyLastUsageTime() + 10000 - time())));
     return ['status' => $status, 'html' => preg_replace('|charset="?windows-1251"?|u', 'charset="utf-8"', $html)];
 }