/** * @return null|string The job to be executed, or null if there is no job and we can terminate. */ function work_getOneForMe() { // get redis $redis = CacheMachine::getPRedisClientOrDie(); // test if we can execute $newCount = $redis->incr(JOBS_COUNT_NAME); // if we can't, just give up if ($newCount <= JOBS_COUNT_MAX) { // if we can, get the job and leave the state as 'incremented' $job = $redis->lpop(JOBS_QUEUE_NAME); if ($job != null) { // also save it to the recent pipe for inspection $redis->lpush(JOBS_RECENT_NAME, $job); $redis->ltrim(JOBS_RECENT_NAME, 0, JOBS_RECENT_SIZE - 1); // also save it to the global set of queries $redis->sadd(JOBS_DONE_SET_NAME, [$job]); // and yes, return the job to be executed :) return $job; } } // if no job, or no slots.. consider it done :) work_doneWithMyCurrent(); return null; }
continue; } CacheMachine::addToSet(VIDEOS_PROCESSED_SET, $video->videoId); // resolve the captions, and skip if failed if (!$video->resolveCaptions()) { echo 'C(' . ltrim($video->getLastCaptionIssue(), ' ') . ')'; continue; } // also resolve details: numbers of views, etc. $video->resolveDetails(); if ($video->countViews < MIN_VIDEO_VIEWS) { echo 'D(v-' . $video->countViews . ')'; continue; } // send it to the Index (to be indexed) if (!$indexMachine->addOrUpdate($video->videoId, $video)) { echo 'I'; continue; } // video processed, all details are present, subtitles downloaded and indexed array_push($newVideos, $video); //CacheMachine::addToSet(VIDEOS_INDEXED_SET_NAME, $video->videoId); echo '.'; } echo "]\n"; echo $outPrefix . sizeof($newVideos) . " added to the index\n"; // done work_doneWithMyCurrent(); } while ($workingQuery = work_getOneForMe()); // all done for me echo "\n";