コード例 #1
0
ファイル: Monster.php プロジェクト: lucy-udalykh/samples
 public static function endExportProcess($data = null)
 {
     if (!empty($data) && $data->key == config('monster.key')) {
         $message = 'Total records: ' . number_format($data->jobs_count) . ' Processed records: ' . number_format($data->processed_count);
         $process = $data->source_id;
         ExportLog::create(['process' => $process, 'filename' => $data->file_name, 'message' => $message]);
         self::moveFtpFile($data->file_name, $process);
     }
     return ['success' => 'true'];
 }
コード例 #2
0
ファイル: Kernel.php プロジェクト: lucy-udalykh/samples
 /**
  * Define the application's command schedule.
  *
  * @param  \Illuminate\Console\Scheduling\Schedule  $schedule
  * @return void
  */
 protected function schedule(Schedule $schedule)
 {
     $schedule->command('export-monster-files test')->cron('00 * * * *')->name('process-feed')->withoutOverlapping();
     $schedule->call(function () {
         $files = \App\FileDeleteQueue::all();
         foreach ($files as $file) {
             $local_file = config('monster.tmp_storage_path') . $file->file_name;
             if (file_exists($local_file)) {
                 $tmp_is_deleted = unlink(realpath($local_file));
                 if (!$tmp_is_deleted) {
                     \App\ExportLog::create(['process' => 'deleting local file', 'filename' => $ftp_file, 'message' => 'cannot be deleted from /tmp/ folder']);
                 } else {
                     $file->delete();
                 }
             }
         }
     })->name('delete-local-files')->twiceDaily(12, 23);
     $schedule->call(function () {
         // deleting expired jobs
         $today = date('Y-m-d', time());
         $jobs = \App\Job::where('end_date', '<', $today)->take(1000)->get();
         if (!$jobs->isEmpty()) {
             $i = 0;
             foreach ($jobs as $j) {
                 $j->delete();
                 $i++;
             }
             \App\ExportLog::create(['process' => 'deleting expired jobs', 'message' => 'Deleted ' . $i . ' jobs']);
         }
     })->cron('5 * * * *')->name('delete-expired-jobs-hourly')->withoutOverlapping();
     $schedule->call(function () {
         $today = date('Y-m-d', time() - 3 * 86400);
         \App\ExportRecordFailure::where('created_at', '<', $today)->take(3000)->delete();
     })->cron('5 * * * *')->name('delete-export-record-failures')->withoutOverlapping();
     $schedule->call(function () {
         $queue = \App\CacheQueue::where(['in_process' => 0])->take(10);
         $records = $queue->get();
         if (!$records->isEmpty()) {
             $queue->update(['in_process' => 1]);
             foreach ($records as $record) {
                 $run = call_user_func($record->model . '::' . $record->method, $record->args);
                 if ($run) {
                     $record->delete();
                 } else {
                     $record->in_process = null;
                     $record->save();
                 }
             }
         }
     })->everyMinute()->name('process-cache-queue-records');
 }
コード例 #3
0
 /**
  * Execute the command.
  *
  *  php artisan export-monster-files  ftp-folder 
  *
  * @return void
  */
 public function handle()
 {
     $ftp_dir = $this->argument('ftp-folder') . '/';
     $site_domain = config('app.domain');
     $process = $source_id = $this->argument('ftp-folder');
     $filesInProgress = \App\FileInProgress::all()->count();
     if ($filesInProgress > 0) {
         return;
     }
     // connecting to monster ftp server
     $ftp_conn = ftp_connect(config('monster.ftp_server'), 21);
     if ($ftp_conn) {
         $ftp_login = ftp_login($ftp_conn, config('monster.ftp_user'), config('monster.ftp_password'));
         if ($ftp_login) {
             $is_passive = ftp_pasv($ftp_conn, true);
             if ($is_passive) {
                 $is_dir_changed = ftp_chdir($ftp_conn, $ftp_dir);
                 if (!$is_dir_changed) {
                     ftp_close($ftp_conn);
                     return;
                 } else {
                     // Generate ftp cache list
                     $ftp_files = ftp_nlist($ftp_conn, "");
                     if (!empty($ftp_files)) {
                         // remove directories from the list
                         $ftp_files = cleanupFtpDir($ftp_files);
                         if (empty($ftp_files)) {
                             ftp_close($ftp_conn);
                             return;
                         }
                         // get the first one
                         $ftp_file = $ftp_files[0];
                         if ($filesInProgress == 0) {
                             \App\FileInProgress::create(['file_name' => $ftp_file]);
                         } else {
                             ftp_close($ftp_conn);
                             return;
                         }
                         $tmp_file_location = config('monster.tmp_storage_path') . $ftp_file;
                         if (ftp_get($ftp_conn, $tmp_file_location, $ftp_file, FTP_BINARY)) {
                             $file_content = file_exists($tmp_file_location);
                             if ($file_content !== false) {
                                 $filesize = filesize($tmp_file_location);
                                 if (!empty($filesize)) {
                                     $filesize = formatFileSize($filesize);
                                 }
                                 \App\ExportLog::create(['process' => $process, 'filename' => $ftp_file, 'filesize' => $filesize, 'message' => 'Starting the process ....']);
                                 $url = $site_domain . ':8080/node';
                                 $args = ['key' => config('monster.key'), 'source_id' => $source_id, 'file_name' => $ftp_file, 'submit_host' => $site_domain, 'submit_path' => '/monster/create_job_record', 'done_path' => '/monster/end_job_process'];
                                 $result = json_decode(Curl::post($url, [], $args));
                                 //File::append('/tmp/monster.txt', "result from curl: $result \n");
                                 if (!empty($result->error)) {
                                     $err_msg = $result->error;
                                     if ($result->error == 'missing data') {
                                         $err_msg = $result->error . ': ' . $result->req_data;
                                     }
                                     \App\ExportLog::create(['process' => $process, 'filename' => $ftp_file, 'message' => $err_msg]);
                                     \App\Monster::moveFtpFile($ftp_file, $process);
                                 }
                             } else {
                                 \App\ExportLog::create(['process' => $process, 'filename' => $ftp_file, 'message' => 'Cannot save file locally']);
                             }
                         } else {
                             \App\ExportLog::create(['process' => $process, 'filename' => $ftp_file, 'message' => 'Cannot download file']);
                         }
                     }
                 }
             }
         }
         ftp_close($ftp_conn);
     }
 }