Пример #1
0
 /**
  * Execute console command.
  */
 public function handle()
 {
     if ($this->option('single')) {
         $output = [];
         exec('pgrep -l -f "^php (.*?)artisan start"', $output);
         foreach ($output as $line) {
             $pid = preg_replace('|([0-9]+)(\\s.*)|u', '$1', $line);
             if ($pid != getmypid()) {
                 exec("kill -9 {$pid}");
             }
         }
     }
     if ($this->option('proxies')) {
         $this->workers = $this->option('proxies');
         $this->proxyManager->useProxy(true);
     }
     if ($this->proxyManager->useProxy()) {
         $this->proxyManager->connect($this->workers, $this->option('kill_old_proxies'));
     }
     if ($job_id = $this->option('job')) {
         $job = Job::find($job_id);
         if ($job) {
             $job->execute();
         } else {
             _log("Job {$job_id} is not found.");
         }
         return;
     }
     if (!$this->jobsManager->count()) {
         _log('No jobs found. Initializing a new discovery and download jobs.');
         $this->discoverer->discoverNewLaws();
         $this->downloader->downloadNewLaws();
     }
     $this->jobsManager->launch($this->workers);
 }
Пример #2
0
 /**
  * Execute console command.
  */
 public function handle()
 {
     if ($this->option('jobs') || $this->option('all')) {
         $this->jobsManager->deleteAll();
     }
     if ($this->option('proxies') || $this->option('all')) {
         $this->proxy->reset();
     }
     return true;
 }
Пример #3
0
 /**
  * Validate download result.
  *
  * @param $path
  * @param $html
  * @param $status
  * @param $options
  *
  * @return bool
  * @throws Exceptions\DocumentCantBeDownloaded
  * @throws Exceptions\DocumentHasErrors
  * @throws Exceptions\DocumentIsMissing
  * @throws Exceptions\ProxyBanned
  * @throws Exceptions\UnknownProblem
  */
 protected function validate($path, &$html, &$status, $options)
 {
     $opendata = isset($options['opendata']) && $options['opendata'] ? true : false;
     // access denied
     if ($status == 403 || $this->detectFakeContent($html, '403')) {
         $this->proxyManager->banProxy();
         $this->download_error($path, $html, new Exceptions\ProxyBanned($this->proxyManager->getProxyIp()));
     }
     // document is missing or server might be down
     if (in_array($status, [204, 400, 404, 500, 502]) || $this->detectFakeContent($html, '404')) {
         $hasMoreIdentities = $this->identity->switchIdentity();
         if ($hasMoreIdentities) {
             $status = $status != 200 ? $status : 204;
             return false;
         } else {
             $this->download_error($path, $html, new Exceptions\DocumentIsMissing());
         }
     }
     // status is ok, but document load was not finished
     if (in_array($status, [206]) || strpos($html, '</body>') === false) {
         $status = 206;
         return false;
     }
     // status is ok, but document content has errors
     if ($errors = $this->detectFakeContent($html, 'error')) {
         $this->download_error($path, $html, new Exceptions\DocumentHasErrors($errors));
     }
     // status is ok, but document JS protected
     if ($newUrl = $this->detectJSProtection($html)) {
         $result = $this->doDownload($newUrl, 10, $opendata);
         $html = $result['html'];
         $status = $result['status'];
         if ($this->detectJSProtection($html)) {
             $this->download_error($path, $html, new Exceptions\DocumentCantBeDownloaded('Strong JS protection.'));
         }
         // do a second validation run on fresh content.
         return $this->validate($path, $html, $status, $options);
     }
     if (!in_array($status, [200, 300, 301, 302, 303, 304, 307, 408])) {
         $this->download_error($path, $html, new Exceptions\UnknownProblem("Download status is {$status}.", $this->shortURL($options['url']), isset($html) ? $html : '{NO DATA}'));
     }
     return true;
 }
Пример #4
0
 public function realWorkersCount($workers_count)
 {
     return env('OFFLINE_PRIORITY', false) ? $workers_count : min($workers_count, $this->proxyManager->count());
 }