/**
* Execute console command.
*/
public function handle()
{
if ($this->option('jobs') || $this->option('all')) {
$this->jobsManager->deleteAll();
}
if ($this->option('proxies') || $this->option('all')) {
$this->proxy->reset();
}
return true;
}
/**
* Execute console command.
*/
public function handle()
{
if ($this->option('single')) {
$output = [];
exec('pgrep -l -f "^php (.*?)artisan start"', $output);
foreach ($output as $line) {
$pid = preg_replace('|([0-9]+)(\\s.*)|u', '$1', $line);
if ($pid != getmypid()) {
exec("kill -9 {$pid}");
}
}
}
if ($this->option('proxies')) {
$this->workers = $this->option('proxies');
$this->proxyManager->useProxy(true);
}
if ($this->proxyManager->useProxy()) {
$this->proxyManager->connect($this->workers, $this->option('kill_old_proxies'));
}
if ($job_id = $this->option('job')) {
$job = Job::find($job_id);
if ($job) {
$job->execute();
} else {
_log("Job {$job_id} is not found.");
}
return;
}
if (!$this->jobsManager->count()) {
_log('No jobs found. Initializing a new discovery and download jobs.');
$this->discoverer->discoverNewLaws();
$this->downloader->downloadNewLaws();
}
$this->jobsManager->launch($this->workers);
}
/**
* Crawl the law list page. Take all law urls from it and add them to database.
*
* @param string $law_list_url Law list URL.
* @param string $date
* @param int $page_num
* @param bool $re_download
*/
public function discoverDailyLawListPage($law_list_url, $page_num, $date, $re_download = false)
{
$data = downloadList($law_list_url, ['re_download' => $page_num > 1 ? $re_download || $this->re_download : false, 'save' => $date != date('Y-m-d')]);
foreach ($data['laws'] as $id => $law) {
Law::firstOrCreate(['id' => $id])->update(['date' => $law['date']]);
$this->jobsManager->add('command.lawgrabber.download', 'downloadCard', ['id' => $id], 'download', 1);
}
}
/**
* Download a specific law's card page.
*
* @param string $id Law ID.
* @param bool $re_download Whether or not to re-download card page.
*
* @return Law
* @throws JobChangePriorityException
* @throws Exceptions\ProxyBanned
*/
function downloadCard($id, $re_download = false)
{
/**
* @var $law Law
*/
$law = Law::find($id);
try {
$card = downloadCard($id, ['re_download' => $re_download || $this->re_download, 'check_related' => $law->status == Law::NOT_DOWNLOADED && !max_date()]);
} catch (Exceptions\ProxyBanned $e) {
throw $e;
} catch (\Exception $e) {
$message = str_replace('ShvetsGroup\\Service\\Exceptions\\', '', get_class($e)) . ($e->getMessage() ? ': ' . $e->getMessage() : '');
throw new JobChangePriorityException($message, -15);
}
DB::transaction(function () use($law, $card) {
$law->card = $card['card'];
$law->title = $card['title'];
$law->date = $card['date'];
$law->setIssuers($card['meta'][Issuer::FIELD_NAME]);
$law->setTypes($card['meta'][Type::FIELD_NAME]);
$law->state = isset($card['meta'][State::FIELD_NAME]) ? reset($card['meta'][State::FIELD_NAME]) : State::STATE_UNKNOWN;
$law->has_text = $card['has_text'] ? $law->has_text = Law::HAS_TEXT : ($law->has_text = Law::NO_TEXT);
$has_unknown_revision = false;
foreach ($card['revisions'] as &$revision) {
if ($revision['date'] == '??.??.????') {
$has_unknown_revision = true;
continue;
}
$data = ['law_id' => $revision['law_id'], 'date' => $revision['date'], 'comment' => $revision['comment']];
if ($law->notHasText() || isset($revision['no_text']) && $revision['no_text'] && $revision['date'] != $card['active_revision']) {
$data['status'] = Revision::NO_TEXT;
$data['text'] = '';
}
$r = Revision::findROrNew($data['law_id'], $data['date']);
$r->save();
$r->update($data);
}
// We should update revision which has just come into power.
if ($law->active_revision && $law->active_revision != $card['active_revision']) {
Revision::find($data['law_id'], $card['active_revision'])->update(['status' => Revision::NEEDS_UPDATE]);
}
$law->active_revision = $card['active_revision'];
foreach ($law->revisions()->where('status', Revision::NEEDS_UPDATE)->get() as $revision) {
$this->jobsManager->add('command.lawgrabber.download', 'downloadRevision', ['law_id' => $revision->law_id, 'date' => $revision->date], 'download', $revision->date == $law->active_revision ? 0 : -1);
}
if (isset($card['changes_laws']) && $card['changes_laws']) {
Law::where('id', array_column($card['changes_laws'], 'id'))->update(['status' => Law::DOWNLOADED_BUT_HAS_UNKNOWN_REVISION]);
foreach ($card['changes_laws'] as $l) {
$this->jobsManager->add('command.lawgrabber.download', 'downloadCard', ['id' => $l['id'], 're_download' => true], 'download', 2);
}
}
$law->card_updated = $card['timestamp'];
$law->status = $has_unknown_revision ? Law::DOWNLOADED_BUT_HAS_UNKNOWN_REVISION : Law::UP_TO_DATE;
$law->save();
});
return $law;
}
