public static function boot() { parent::boot(); static::creating(function ($workerunit) { // dd($workerunit); // Inherit documentType, domain and format if (empty($workerunit->documentType)) { $j = Job::where('_id', $workerunit->job_id)->first(); $workerunit->documentType = $j->documentType; } // transform answer into annotation vector to prepare for the CrowdTruth metrics $workerunit->annotationVector = $workerunit->createAnnotationVector(); // Activity if not exists if (empty($workerunit->activity_id)) { try { $activity = new Activity(); $activity->label = "Workerunit is saved."; $activity->softwareAgent_id = $workerunit->softwareAgent_id; $activity->save(); $workerunit->activity_id = $activity->_id; Log::debug("Saving workerunit {$workerunit->_id} with activity {$workerunit->activity_id}."); } catch (Exception $e) { if ($activity) { $activity->forceDelete(); } //if($workerunit) $workerunit->forceDelete(); throw new Exception('Error saving activity for workerunit.'); } } }); }
public function fire($job, $data) { // Metrics set_time_limit(600); $workerunit = unserialize($data['workerunit']); $spammer = Job::where('_id', $workerunit->job_id)->whereIn('metrics.spammers.list', [$workerunit->crowdAgent_id])->exists(); $workerunit->spam = $spammer; $workerunit->save(); \Log::debug("Updated Workerunit {$workerunit->_id}."); $job->delete(); }
/** * @param $data = array(unitids) */ public function fire($job, $data) { foreach ($data as $id) { set_time_limit(600); $unit = Entity::id($id)->first(); $batches = count(Batch::whereIn('parents', [$unit->_id])->get()->toArray()); $workerunit = array('count' => 0, 'spam' => 0, 'nonSpam' => 0); $workerlist = $workersspam = $workersnonspam = $joblist = array(); foreach (Workerunit::where('unit_id', $unit->_id)->get() as $a) { $joblist[] = $a->job_id; $workerlist[] = $a->crowdAgent_id; if ($a->spam) { $workerunit['spam']++; $workersspam[] = $a->crowdAgent_id; } else { $workerunit['nonSpam']++; $workersnonspam[] = $a->crowdAgent_id; } } $workerunit['count'] = $workerunit['spam'] + $workerunit['nonSpam']; $workers['count'] = count(array_unique($workerlist)); $workers['spam'] = count(array_unique($workersspam)); $workers['nonSpam'] = count(array_unique($workersnonspam)); $workers['potentialSpam'] = count(array_intersect($workersspam, $workersnonspam)); $platformField = array(); $platformField['cf'] = count(Entity::where('unit_id', $unit->_id)->where('softwareAgent_id', 'cf')->get()->toArray()); $platformField['amt'] = count(Entity::where('unit_id', $unit->_id)->where('softwareAgent_id', 'amt')->get()->toArray()); //filtered $filteredField = array(); $filteredField['job_ids'] = array_flatten(Job::where('metrics.filteredUnits.list', 'all', array($unit['_id']))->get(['_id'])->toArray()); $filteredField['count'] = count($filteredField['job_ids']); $derivatives = Entity::whereIn('parents', array($unit->_id))->lists('_id'); $children["count"] = count($derivatives); $children["list"] = $derivatives; $unit->cache = ["jobs" => 1, "workers" => $workers, "softwareAgent" => $platformField, "workerunits" => $workerunit, "filtered" => $filteredField, "batches" => $batches, "children" => $children]; $avg_clarity = Entity::where('metrics.units.withoutSpam.' . $unit->_id, 'exists', 'true')->avg('metrics.units.withoutSpam.' . $unit->id . '.avg.max_relation_Cos'); if (!isset($avg_clarity)) { $avg_clarity = 0; } $unit->avg_clarity = $avg_clarity; $unit->update(); \Log::debug("Updated unit {$unit->_id}."); } $job->delete(); // the Queue job... }
public function updateStats2() { // take all the jobs for that worker // TODO: change to Job:: if ($crowdAgentJobs = Job::where('metrics.workers.withFilter.' . $this->_id, 'exists', true)->get(['_id'])) { //if there is at least one job with that worker if (count($crowdAgentJobs->toArray()) > 0) { $spam = $nonspam = $totalNoOfWorkerunits = 0; foreach ($this->workerunits as $a) { $totalNoOfWorkerunits++; if ($a->spam) { $spam++; } else { $nonspam++; } $types[] = $a->type; $jobids[] = $a->job_id; $unitids[] = $a->unit_id; } // $this->WorkerunitStats = array('count'=>$total['count'], 'spam'=>$spam, 'nonspam'=>$nonspam); $distinctWorkerunitTypes = array_unique($types); // These actually are the Workerunit types $workerParticipatedIn = count(array_unique($unitids)); dd($workerParticipatedIn); $cache["workerunits"] = ["count" => $totalNoOfWorkerunits, "spam" => $spam, "nonspam" => $nonspam]; // take all distinct batches $distinctBatchIds = Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->distinct('batch_id')->get(['_id']); /* foreach($distinctBatchIds as $distinctBatchId) { $batchParents = array_flatten(Entity::where('_id', '=', $distinctBatchId[0])->lists('parents')); //print_r($batchParents[0]); $batchParentsType = Entity::where('_id', '=', $batchParents[0])->distinct('documentType')->get(['documentType']); //print_r(array_flatten($batchParentsType->toArray())[0]); if(isset($cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]])) { $cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]] = $cache["mediaTypes"][array_flatten($batchParentsType->toArray())[0]] + 1; } else { $cache["mediaTypes"]["types"] = []; $cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]] = 1; } } $cache["mediaTypes"]["distinct"] = sizeof(array_keys($cache["mediaTypes"]["types"])); */ if (count($distinctWorkerunitTypes) > 0) { $cache["jobTypes"] = ["distinct" => count($distinctWorkerunitTypes), "count" => count(array_unique($jobids)), "types" => []]; foreach ($distinctWorkerunitTypes as $distinctJobType) { $distinctJobTypeCount = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->count(); $distinctJobTemplateTypes = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->distinct('template')->get()->toArray(); $countJobTemplateTypes = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->count(); //$cache["jobTypes"]["types"][$distinctJobType[0]] = []; $cache["jobTypes"]["types"][$distinctJobType]['distinct'] = count($distinctJobTemplateTypes); $cache["jobTypes"]["types"][$distinctJobType]['count'] = count($countJobTemplateTypes); $cache["jobTypes"]["types"][$distinctJobType]["templates"] = []; foreach ($distinctJobTemplateTypes as $distinctJobTemplateType) { $distinctJobTemplateAndCount = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->where('template', $distinctJobTemplateType)->count(); $cache["jobTypes"]["types"][$distinctJobType]["templates"][$distinctJobTemplateType[0]] = $distinctJobTemplateAndCount; } } } $jobsAsSpammer = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->whereIn('metrics.spammers.list', [$this->_id])->lists('platformJobId'); $cache["spammer"]["count"] = count($jobsAsSpammer); $cache["spammer"]["jobs"] = array_flatten($jobsAsSpammer); $this->cache = $cache; $this->save(); } else { $this->save(); } } }
public function postSubmitFinal($ordersandbox = 'order') { $batch = unserialize(Session::get('batch')); $batchColumnsNewTemplate = array(); $batchColumnsExtraChosenTemplate = array(); $associationsTemplBatch = array(); $ownTemplate = false; // Use existing job configuration content (if available) if (!($jc = unserialize(Session::get('jobconf')))) { $jc = new JobConfiguration(); $jc->type = "jobconf"; $jcco = array(); } else { $jcco = $jc->content; } $own = false; if (Input::has('templateTypeOwn') && strlen(Input::get('templateTypeOwn')) > 0) { $jcco['type'] = Input::get('templateTypeOwn'); $batchColumns = Input::get('batchColumns'); $newNamesForBatchColumns = Input::get('newcolnames'); $newNamesForBatchColumnsArray = explode(",", $newNamesForBatchColumns); array_pop($newNamesForBatchColumnsArray); if ($batchColumns == null) { return Redirect::back()->with('flashError', "You did not choose the batch columns"); } else { foreach ($newNamesForBatchColumnsArray as $value) { $oldNewArray = explode(" - ", $value); if (in_array($oldNewArray[0], $batchColumns)) { $batchColumnsNewTemplate[$oldNewArray[0]] = $oldNewArray[1]; } } } $ownTemplate = true; $own = true; } else { $jcco['type'] = Input::get('templateType'); if ($jcco['type'] == null) { return Redirect::back()->with('flashError', "You did not fill in the type of the template"); } $jcco['type'] = Input::get('templateType'); if ($jcco['type'] == Null) { return Redirect::back()->with('flashError', "You did not fill in the type of the template"); } // get a selected, newest jcbase $maxi = Template::where("type", $jcco['type'])->max('version'); $jcbase = Template::where("type", $jcco['type'])->where('version', $maxi)->first(); if (!isset($jcbase)) { Session::flash('flashError', "template not found: " . $jcco['type']); return Redirect::to("jobs2/submit"); } if (!isset($jcbase['cml'])) { // Template must have CML field Session::flash('flashError', "No template details in this template"); return Redirect::to("jobs2/submit"); } $batchColumnsExtraChosenTemplate = Input::get('addMoreColumns'); $fieldsInChosenTemplate = Input::get('tempFields'); $arrayFields = explode(",", $fieldsInChosenTemplate); foreach ($arrayFields as $field) { $association = Input::get($field); array_push($associationsTemplBatch, $field . " - " . $association); if ($association == null || $association == "---") { return Redirect::back()->with('flashError', "You did not fill in all the associations for the template fields"); } } $jcco['cml'] = $jcbase['cml']; if (isset($jcbase['css'])) { $jcco['css'] = $jcbase['css']; } if (isset($jcbase['instructions'])) { $jcco['instructions'] = $jcbase['instructions']; } if (isset($jcbase['js'])) { $jcco['js'] = $jcbase['js']; } $jcco['template_id'] = $jcbase['_id']; } if (Input::has('titleOwn') && strlen(Input::get('titleOwn')) > 0) { $jcco['title'] = Input::get('titleOwn'); } else { $jcco['title'] = Input::get('title'); } if ($jcco['title'] == Null) { return Redirect::back()->with('flashError', "You did not fill in the title of the template"); } $jcco['platform'] = array("DrDetectiveGamingPlatform"); //TODOJORAN $jcco['description'] = Input::get('description'); $jcco['title'] = $jcco['title'] . "[[" . $jcco['type'] . "(" . $batch->_id . ", " . $batch->domain . ", " . $batch->format . ")]]"; ///////// PUT $jcco['platform'] = array("cf2"); $jc->content = $jcco; if ($own) { $_tt = Template::where('type', $jcco['type'])->where("format", $batch->format)->first(); if (isset($_tt)) { Session::flash('flashError', "There is already a template of this type. Please rename (or select this template from dropdown list."); return Redirect::to("jobs2/submit"); } } try { // Save activity $activity = new Activity(); $activity->label = "Job is uploaded to crowdsourcing platform."; $activity->softwareAgent_id = 'jobcreator'; // JOB softwareAgent_id = $platform. Does this need to be the same? $activity->save(); // Save jobconf if necessary $hash = md5(serialize($jc->content)); if ($existingid = JobConfiguration::where('hash', $hash)->pluck('_id')) { //[qq] $jcid = $existingid; // Don't save, it already exists. } else { $jc->format = $batch->format; $jc->domain = $batch->domain; $jc->project = $batch->project; $jc->user_id = $batch->user_id; $jc->softwareAgent_id = $activity->softwareAgent_id; $jc->hash = $hash; $jc->activity_id = $activity->_id; $jc->save(); $jcid = $jc->_id; } // Software Agent ID which will perform the job // TODO: Fix this -- choose one of the possibly many platforms in $jcco // at the moment we are picking the first arbitrarily $job_sw_agent = $jcco['platform'][0]; // Publish jobs $j = new Job(); $j->format = $batch->format; $j->project = $batch->project; $j->domain = $batch->domain; $j->user_id = $batch->user_id; $j->type = "job"; $j->batch_id = $batch->_id; $j->jobConf_id = $jcid; $j->softwareAgent_id = $job_sw_agent; $j->activity_id = $activity->_id; $j->iamemptyjob = "yes"; $extraInfoBatch = array(); $extraInfoBatch["batchColumnsNewTemplate"] = $batchColumnsNewTemplate; $extraInfoBatch["batchColumnsExtraChosenTemplate"] = $batchColumnsExtraChosenTemplate; $extraInfoBatch["associationsTemplBatch"] = $associationsTemplBatch; $extraInfoBatch["ownTemplate"] = $ownTemplate; $j->extraInfoBatch = $extraInfoBatch; $j->save(); //convert to publish later $j->publish($ordersandbox == 'sandbox' ? true : false); $jobs[] = $j; $successmessage = "Created job with jobConf :-)"; // $platform = App::make($job_sw_agent); $platform = App::make("cf2"); // $platform = App::make("DrDetectiveGamingPlatform"); //TODOJORAN $platform->refreshJob($j->_id); Session::flash('flashSuccess', $successmessage); return Redirect::to("jobs"); } catch (Exception $e) { // Undo creation and delete jobs // if(isset($jobs)) // foreach($jobs as $j){ // if(isset($j->platformJobId)) // $j->undoCreation($j->platformJobId); // $j->forceDelete(); // } //delete activity if (isset($j)) { $j->forceDelete(); } if (isset($jc)) { $jc->forceDelete(); } if ($activity) { $activity->forceDelete(); } throw $e; //for debugging Session::flash('flashError', $e->getMessage()); return Redirect::to("jobs2/submit"); } }
/** * Create job entity * Note: the job does not get a platformJobId because this is unknown */ public function createJob($config, $activity, $batch, $settings) { $entity = new Job(); $entity->_id = $entity->_id; $entity->batch_id = $batch->_id; $entity->project = $settings['project']; $entity->documentType = $settings['documentType']; $entity->resultType = $settings['resultType']; $entity->type = "job"; $entity->completion = 1; $entity->expectedWorkerUnitsCount = 450; $entity->finishedAt = new MongoDate(); $entity->jobConf_id = $config; $entity->platformJobId = $settings['filename']; $entity->projectedCost = 12.0; $entity->realCost = 11.97; $entity->runningTimeInSeconds = 190714; $entity->softwareAgent_id = $settings['platform']; $entity->startedAt = new MongoDate(); $entity->status = "imported"; $entity->template = "imported"; $entity->activity_id = $activity; $entity->save(); array_push($this->status['success'], "Job created (" . $entity->_id . ")"); return $entity; }