Exemple #1
2
 /**
  * Download a specific law's card page.
  *
  * @param string $id          Law ID.
  * @param bool   $re_download Whether or not to re-download card page.
  *
  * @return Law
  * @throws JobChangePriorityException
  * @throws Exceptions\ProxyBanned
  */
 function downloadCard($id, $re_download = false)
 {
     /**
      * @var $law Law
      */
     $law = Law::find($id);
     try {
         $card = downloadCard($id, ['re_download' => $re_download || $this->re_download, 'check_related' => $law->status == Law::NOT_DOWNLOADED && !max_date()]);
     } catch (Exceptions\ProxyBanned $e) {
         throw $e;
     } catch (\Exception $e) {
         $message = str_replace('ShvetsGroup\\Service\\Exceptions\\', '', get_class($e)) . ($e->getMessage() ? ': ' . $e->getMessage() : '');
         throw new JobChangePriorityException($message, -15);
     }
     DB::transaction(function () use($law, $card) {
         $law->card = $card['card'];
         $law->title = $card['title'];
         $law->date = $card['date'];
         $law->setIssuers($card['meta'][Issuer::FIELD_NAME]);
         $law->setTypes($card['meta'][Type::FIELD_NAME]);
         $law->state = isset($card['meta'][State::FIELD_NAME]) ? reset($card['meta'][State::FIELD_NAME]) : State::STATE_UNKNOWN;
         $law->has_text = $card['has_text'] ? $law->has_text = Law::HAS_TEXT : ($law->has_text = Law::NO_TEXT);
         $has_unknown_revision = false;
         foreach ($card['revisions'] as &$revision) {
             if ($revision['date'] == '??.??.????') {
                 $has_unknown_revision = true;
                 continue;
             }
             $data = ['law_id' => $revision['law_id'], 'date' => $revision['date'], 'comment' => $revision['comment']];
             if ($law->notHasText() || isset($revision['no_text']) && $revision['no_text'] && $revision['date'] != $card['active_revision']) {
                 $data['status'] = Revision::NO_TEXT;
                 $data['text'] = '';
             }
             $r = Revision::findROrNew($data['law_id'], $data['date']);
             $r->save();
             $r->update($data);
         }
         // We should update revision which has just come into power.
         if ($law->active_revision && $law->active_revision != $card['active_revision']) {
             Revision::find($data['law_id'], $card['active_revision'])->update(['status' => Revision::NEEDS_UPDATE]);
         }
         $law->active_revision = $card['active_revision'];
         foreach ($law->revisions()->where('status', Revision::NEEDS_UPDATE)->get() as $revision) {
             $this->jobsManager->add('command.lawgrabber.download', 'downloadRevision', ['law_id' => $revision->law_id, 'date' => $revision->date], 'download', $revision->date == $law->active_revision ? 0 : -1);
         }
         if (isset($card['changes_laws']) && $card['changes_laws']) {
             Law::where('id', array_column($card['changes_laws'], 'id'))->update(['status' => Law::DOWNLOADED_BUT_HAS_UNKNOWN_REVISION]);
             foreach ($card['changes_laws'] as $l) {
                 $this->jobsManager->add('command.lawgrabber.download', 'downloadCard', ['id' => $l['id'], 're_download' => true], 'download', 2);
             }
         }
         $law->card_updated = $card['timestamp'];
         $law->status = $has_unknown_revision ? Law::DOWNLOADED_BUT_HAS_UNKNOWN_REVISION : Law::UP_TO_DATE;
         $law->save();
     });
     return $law;
 }
Exemple #2
0
 /**
  * Execute console command.
  */
 public function handle()
 {
     if ($this->option('jobs') || $this->option('all')) {
         $this->jobsManager->deleteAll();
     }
     if ($this->option('proxies') || $this->option('all')) {
         $this->proxy->reset();
     }
     return true;
 }
Exemple #3
0
 /**
  * Execute console command.
  */
 public function handle()
 {
     if ($this->option('single')) {
         $output = [];
         exec('pgrep -l -f "^php (.*?)artisan start"', $output);
         foreach ($output as $line) {
             $pid = preg_replace('|([0-9]+)(\\s.*)|u', '$1', $line);
             if ($pid != getmypid()) {
                 exec("kill -9 {$pid}");
             }
         }
     }
     if ($this->option('proxies')) {
         $this->workers = $this->option('proxies');
         $this->proxyManager->useProxy(true);
     }
     if ($this->proxyManager->useProxy()) {
         $this->proxyManager->connect($this->workers, $this->option('kill_old_proxies'));
     }
     if ($job_id = $this->option('job')) {
         $job = Job::find($job_id);
         if ($job) {
             $job->execute();
         } else {
             _log("Job {$job_id} is not found.");
         }
         return;
     }
     if (!$this->jobsManager->count()) {
         _log('No jobs found. Initializing a new discovery and download jobs.');
         $this->discoverer->discoverNewLaws();
         $this->downloader->downloadNewLaws();
     }
     $this->jobsManager->launch($this->workers);
 }
Exemple #4
0
 /**
  * Crawl the law list page. Take all law urls from it and add them to database.
  *
  * @param string $law_list_url Law list URL.
  * @param string $date
  * @param int $page_num
  * @param bool $re_download
  */
 public function discoverDailyLawListPage($law_list_url, $page_num, $date, $re_download = false)
 {
     $data = downloadList($law_list_url, ['re_download' => $page_num > 1 ? $re_download || $this->re_download : false, 'save' => $date != date('Y-m-d')]);
     foreach ($data['laws'] as $id => $law) {
         Law::firstOrCreate(['id' => $id])->update(['date' => $law['date']]);
         $this->jobsManager->add('command.lawgrabber.download', 'downloadCard', ['id' => $id], 'download', 1);
     }
 }