public static function endExportProcess($data = null)
{
if (!empty($data) && $data->key == config('monster.key')) {
$message = 'Total records: ' . number_format($data->jobs_count) . ' Processed records: ' . number_format($data->processed_count);
$process = $data->source_id;
ExportLog::create(['process' => $process, 'filename' => $data->file_name, 'message' => $message]);
self::moveFtpFile($data->file_name, $process);
}
return ['success' => 'true'];
}
/**
* Define the application's command schedule.
*
* @param \Illuminate\Console\Scheduling\Schedule $schedule
* @return void
*/
protected function schedule(Schedule $schedule)
{
$schedule->command('export-monster-files test')->cron('00 * * * *')->name('process-feed')->withoutOverlapping();
$schedule->call(function () {
$files = \App\FileDeleteQueue::all();
foreach ($files as $file) {
$local_file = config('monster.tmp_storage_path') . $file->file_name;
if (file_exists($local_file)) {
$tmp_is_deleted = unlink(realpath($local_file));
if (!$tmp_is_deleted) {
\App\ExportLog::create(['process' => 'deleting local file', 'filename' => $ftp_file, 'message' => 'cannot be deleted from /tmp/ folder']);
} else {
$file->delete();
}
}
}
})->name('delete-local-files')->twiceDaily(12, 23);
$schedule->call(function () {
// deleting expired jobs
$today = date('Y-m-d', time());
$jobs = \App\Job::where('end_date', '<', $today)->take(1000)->get();
if (!$jobs->isEmpty()) {
$i = 0;
foreach ($jobs as $j) {
$j->delete();
$i++;
}
\App\ExportLog::create(['process' => 'deleting expired jobs', 'message' => 'Deleted ' . $i . ' jobs']);
}
})->cron('5 * * * *')->name('delete-expired-jobs-hourly')->withoutOverlapping();
$schedule->call(function () {
$today = date('Y-m-d', time() - 3 * 86400);
\App\ExportRecordFailure::where('created_at', '<', $today)->take(3000)->delete();
})->cron('5 * * * *')->name('delete-export-record-failures')->withoutOverlapping();
$schedule->call(function () {
$queue = \App\CacheQueue::where(['in_process' => 0])->take(10);
$records = $queue->get();
if (!$records->isEmpty()) {
$queue->update(['in_process' => 1]);
foreach ($records as $record) {
$run = call_user_func($record->model . '::' . $record->method, $record->args);
if ($run) {
$record->delete();
} else {
$record->in_process = null;
$record->save();
}
}
}
})->everyMinute()->name('process-cache-queue-records');
}
/**
* Execute the command.
*
* php artisan export-monster-files ftp-folder
*
* @return void
*/
public function handle()
{
$ftp_dir = $this->argument('ftp-folder') . '/';
$site_domain = config('app.domain');
$process = $source_id = $this->argument('ftp-folder');
$filesInProgress = \App\FileInProgress::all()->count();
if ($filesInProgress > 0) {
return;
}
// connecting to monster ftp server
$ftp_conn = ftp_connect(config('monster.ftp_server'), 21);
if ($ftp_conn) {
$ftp_login = ftp_login($ftp_conn, config('monster.ftp_user'), config('monster.ftp_password'));
if ($ftp_login) {
$is_passive = ftp_pasv($ftp_conn, true);
if ($is_passive) {
$is_dir_changed = ftp_chdir($ftp_conn, $ftp_dir);
if (!$is_dir_changed) {
ftp_close($ftp_conn);
return;
} else {
// Generate ftp cache list
$ftp_files = ftp_nlist($ftp_conn, "");
if (!empty($ftp_files)) {
// remove directories from the list
$ftp_files = cleanupFtpDir($ftp_files);
if (empty($ftp_files)) {
ftp_close($ftp_conn);
return;
}
// get the first one
$ftp_file = $ftp_files[0];
if ($filesInProgress == 0) {
\App\FileInProgress::create(['file_name' => $ftp_file]);
} else {
ftp_close($ftp_conn);
return;
}
$tmp_file_location = config('monster.tmp_storage_path') . $ftp_file;
if (ftp_get($ftp_conn, $tmp_file_location, $ftp_file, FTP_BINARY)) {
$file_content = file_exists($tmp_file_location);
if ($file_content !== false) {
$filesize = filesize($tmp_file_location);
if (!empty($filesize)) {
$filesize = formatFileSize($filesize);
}
\App\ExportLog::create(['process' => $process, 'filename' => $ftp_file, 'filesize' => $filesize, 'message' => 'Starting the process ....']);
$url = $site_domain . ':8080/node';
$args = ['key' => config('monster.key'), 'source_id' => $source_id, 'file_name' => $ftp_file, 'submit_host' => $site_domain, 'submit_path' => '/monster/create_job_record', 'done_path' => '/monster/end_job_process'];
$result = json_decode(Curl::post($url, [], $args));
//File::append('/tmp/monster.txt', "result from curl: $result \n");
if (!empty($result->error)) {
$err_msg = $result->error;
if ($result->error == 'missing data') {
$err_msg = $result->error . ': ' . $result->req_data;
}
\App\ExportLog::create(['process' => $process, 'filename' => $ftp_file, 'message' => $err_msg]);
\App\Monster::moveFtpFile($ftp_file, $process);
}
} else {
\App\ExportLog::create(['process' => $process, 'filename' => $ftp_file, 'message' => 'Cannot save file locally']);
}
} else {
\App\ExportLog::create(['process' => $process, 'filename' => $ftp_file, 'message' => 'Cannot download file']);
}
}
}
}
}
ftp_close($ftp_conn);
}
}