public static function boot()
{
parent::boot();
static::creating(function ($workerunit) {
// dd($workerunit);
// Inherit documentType, domain and format
if (empty($workerunit->documentType)) {
$j = Job::where('_id', $workerunit->job_id)->first();
$workerunit->documentType = $j->documentType;
}
// transform answer into annotation vector to prepare for the CrowdTruth metrics
$workerunit->annotationVector = $workerunit->createAnnotationVector();
// Activity if not exists
if (empty($workerunit->activity_id)) {
try {
$activity = new Activity();
$activity->label = "Workerunit is saved.";
$activity->softwareAgent_id = $workerunit->softwareAgent_id;
$activity->save();
$workerunit->activity_id = $activity->_id;
Log::debug("Saving workerunit {$workerunit->_id} with activity {$workerunit->activity_id}.");
} catch (Exception $e) {
if ($activity) {
$activity->forceDelete();
}
//if($workerunit) $workerunit->forceDelete();
throw new Exception('Error saving activity for workerunit.');
}
}
});
}
public function fire($job, $data)
{
// Metrics
set_time_limit(600);
$workerunit = unserialize($data['workerunit']);
$spammer = Job::where('_id', $workerunit->job_id)->whereIn('metrics.spammers.list', [$workerunit->crowdAgent_id])->exists();
$workerunit->spam = $spammer;
$workerunit->save();
\Log::debug("Updated Workerunit {$workerunit->_id}.");
$job->delete();
}
/**
* @param $data = array(unitids)
*/
public function fire($job, $data)
{
foreach ($data as $id) {
set_time_limit(600);
$unit = Entity::id($id)->first();
$batches = count(Batch::whereIn('parents', [$unit->_id])->get()->toArray());
$workerunit = array('count' => 0, 'spam' => 0, 'nonSpam' => 0);
$workerlist = $workersspam = $workersnonspam = $joblist = array();
foreach (Workerunit::where('unit_id', $unit->_id)->get() as $a) {
$joblist[] = $a->job_id;
$workerlist[] = $a->crowdAgent_id;
if ($a->spam) {
$workerunit['spam']++;
$workersspam[] = $a->crowdAgent_id;
} else {
$workerunit['nonSpam']++;
$workersnonspam[] = $a->crowdAgent_id;
}
}
$workerunit['count'] = $workerunit['spam'] + $workerunit['nonSpam'];
$workers['count'] = count(array_unique($workerlist));
$workers['spam'] = count(array_unique($workersspam));
$workers['nonSpam'] = count(array_unique($workersnonspam));
$workers['potentialSpam'] = count(array_intersect($workersspam, $workersnonspam));
$platformField = array();
$platformField['cf'] = count(Entity::where('unit_id', $unit->_id)->where('softwareAgent_id', 'cf')->get()->toArray());
$platformField['amt'] = count(Entity::where('unit_id', $unit->_id)->where('softwareAgent_id', 'amt')->get()->toArray());
//filtered
$filteredField = array();
$filteredField['job_ids'] = array_flatten(Job::where('metrics.filteredUnits.list', 'all', array($unit['_id']))->get(['_id'])->toArray());
$filteredField['count'] = count($filteredField['job_ids']);
$derivatives = Entity::whereIn('parents', array($unit->_id))->lists('_id');
$children["count"] = count($derivatives);
$children["list"] = $derivatives;
$unit->cache = ["jobs" => 1, "workers" => $workers, "softwareAgent" => $platformField, "workerunits" => $workerunit, "filtered" => $filteredField, "batches" => $batches, "children" => $children];
$avg_clarity = Entity::where('metrics.units.withoutSpam.' . $unit->_id, 'exists', 'true')->avg('metrics.units.withoutSpam.' . $unit->id . '.avg.max_relation_Cos');
if (!isset($avg_clarity)) {
$avg_clarity = 0;
}
$unit->avg_clarity = $avg_clarity;
$unit->update();
\Log::debug("Updated unit {$unit->_id}.");
}
$job->delete();
// the Queue job...
}
public function updateStats2()
{
// take all the jobs for that worker
// TODO: change to Job::
if ($crowdAgentJobs = Job::where('metrics.workers.withFilter.' . $this->_id, 'exists', true)->get(['_id'])) {
//if there is at least one job with that worker
if (count($crowdAgentJobs->toArray()) > 0) {
$spam = $nonspam = $totalNoOfWorkerunits = 0;
foreach ($this->workerunits as $a) {
$totalNoOfWorkerunits++;
if ($a->spam) {
$spam++;
} else {
$nonspam++;
}
$types[] = $a->type;
$jobids[] = $a->job_id;
$unitids[] = $a->unit_id;
}
// $this->WorkerunitStats = array('count'=>$total['count'], 'spam'=>$spam, 'nonspam'=>$nonspam);
$distinctWorkerunitTypes = array_unique($types);
// These actually are the Workerunit types
$workerParticipatedIn = count(array_unique($unitids));
dd($workerParticipatedIn);
$cache["workerunits"] = ["count" => $totalNoOfWorkerunits, "spam" => $spam, "nonspam" => $nonspam];
// take all distinct batches
$distinctBatchIds = Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->distinct('batch_id')->get(['_id']);
/*
foreach($distinctBatchIds as $distinctBatchId) {
$batchParents = array_flatten(Entity::where('_id', '=', $distinctBatchId[0])->lists('parents'));
//print_r($batchParents[0]);
$batchParentsType = Entity::where('_id', '=', $batchParents[0])->distinct('documentType')->get(['documentType']);
//print_r(array_flatten($batchParentsType->toArray())[0]);
if(isset($cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]])) {
$cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]] = $cache["mediaTypes"][array_flatten($batchParentsType->toArray())[0]] + 1;
}
else {
$cache["mediaTypes"]["types"] = [];
$cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]] = 1;
}
}
$cache["mediaTypes"]["distinct"] = sizeof(array_keys($cache["mediaTypes"]["types"]));
*/
if (count($distinctWorkerunitTypes) > 0) {
$cache["jobTypes"] = ["distinct" => count($distinctWorkerunitTypes), "count" => count(array_unique($jobids)), "types" => []];
foreach ($distinctWorkerunitTypes as $distinctJobType) {
$distinctJobTypeCount = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->count();
$distinctJobTemplateTypes = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->distinct('template')->get()->toArray();
$countJobTemplateTypes = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->count();
//$cache["jobTypes"]["types"][$distinctJobType[0]] = [];
$cache["jobTypes"]["types"][$distinctJobType]['distinct'] = count($distinctJobTemplateTypes);
$cache["jobTypes"]["types"][$distinctJobType]['count'] = count($countJobTemplateTypes);
$cache["jobTypes"]["types"][$distinctJobType]["templates"] = [];
foreach ($distinctJobTemplateTypes as $distinctJobTemplateType) {
$distinctJobTemplateAndCount = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->where('template', $distinctJobTemplateType)->count();
$cache["jobTypes"]["types"][$distinctJobType]["templates"][$distinctJobTemplateType[0]] = $distinctJobTemplateAndCount;
}
}
}
$jobsAsSpammer = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->whereIn('metrics.spammers.list', [$this->_id])->lists('platformJobId');
$cache["spammer"]["count"] = count($jobsAsSpammer);
$cache["spammer"]["jobs"] = array_flatten($jobsAsSpammer);
$this->cache = $cache;
$this->save();
} else {
$this->save();
}
}
}
public function postSubmitFinal($ordersandbox = 'order')
{
$batch = unserialize(Session::get('batch'));
$batchColumnsNewTemplate = array();
$batchColumnsExtraChosenTemplate = array();
$associationsTemplBatch = array();
$ownTemplate = false;
// Use existing job configuration content (if available)
if (!($jc = unserialize(Session::get('jobconf')))) {
$jc = new JobConfiguration();
$jc->type = "jobconf";
$jcco = array();
} else {
$jcco = $jc->content;
}
$own = false;
if (Input::has('templateTypeOwn') && strlen(Input::get('templateTypeOwn')) > 0) {
$jcco['type'] = Input::get('templateTypeOwn');
$batchColumns = Input::get('batchColumns');
$newNamesForBatchColumns = Input::get('newcolnames');
$newNamesForBatchColumnsArray = explode(",", $newNamesForBatchColumns);
array_pop($newNamesForBatchColumnsArray);
if ($batchColumns == null) {
return Redirect::back()->with('flashError', "You did not choose the batch columns");
} else {
foreach ($newNamesForBatchColumnsArray as $value) {
$oldNewArray = explode(" - ", $value);
if (in_array($oldNewArray[0], $batchColumns)) {
$batchColumnsNewTemplate[$oldNewArray[0]] = $oldNewArray[1];
}
}
}
$ownTemplate = true;
$own = true;
} else {
$jcco['type'] = Input::get('templateType');
if ($jcco['type'] == null) {
return Redirect::back()->with('flashError', "You did not fill in the type of the template");
}
$jcco['type'] = Input::get('templateType');
if ($jcco['type'] == Null) {
return Redirect::back()->with('flashError', "You did not fill in the type of the template");
}
// get a selected, newest jcbase
$maxi = Template::where("type", $jcco['type'])->max('version');
$jcbase = Template::where("type", $jcco['type'])->where('version', $maxi)->first();
if (!isset($jcbase)) {
Session::flash('flashError', "template not found: " . $jcco['type']);
return Redirect::to("jobs2/submit");
}
if (!isset($jcbase['cml'])) {
// Template must have CML field
Session::flash('flashError', "No template details in this template");
return Redirect::to("jobs2/submit");
}
$batchColumnsExtraChosenTemplate = Input::get('addMoreColumns');
$fieldsInChosenTemplate = Input::get('tempFields');
$arrayFields = explode(",", $fieldsInChosenTemplate);
foreach ($arrayFields as $field) {
$association = Input::get($field);
array_push($associationsTemplBatch, $field . " - " . $association);
if ($association == null || $association == "---") {
return Redirect::back()->with('flashError', "You did not fill in all the associations for the template fields");
}
}
$jcco['cml'] = $jcbase['cml'];
if (isset($jcbase['css'])) {
$jcco['css'] = $jcbase['css'];
}
if (isset($jcbase['instructions'])) {
$jcco['instructions'] = $jcbase['instructions'];
}
if (isset($jcbase['js'])) {
$jcco['js'] = $jcbase['js'];
}
$jcco['template_id'] = $jcbase['_id'];
}
if (Input::has('titleOwn') && strlen(Input::get('titleOwn')) > 0) {
$jcco['title'] = Input::get('titleOwn');
} else {
$jcco['title'] = Input::get('title');
}
if ($jcco['title'] == Null) {
return Redirect::back()->with('flashError', "You did not fill in the title of the template");
}
$jcco['platform'] = array("DrDetectiveGamingPlatform");
//TODOJORAN
$jcco['description'] = Input::get('description');
$jcco['title'] = $jcco['title'] . "[[" . $jcco['type'] . "(" . $batch->_id . ", " . $batch->domain . ", " . $batch->format . ")]]";
///////// PUT
$jcco['platform'] = array("cf2");
$jc->content = $jcco;
if ($own) {
$_tt = Template::where('type', $jcco['type'])->where("format", $batch->format)->first();
if (isset($_tt)) {
Session::flash('flashError', "There is already a template of this type. Please rename (or select this template from dropdown list.");
return Redirect::to("jobs2/submit");
}
}
try {
// Save activity
$activity = new Activity();
$activity->label = "Job is uploaded to crowdsourcing platform.";
$activity->softwareAgent_id = 'jobcreator';
// JOB softwareAgent_id = $platform. Does this need to be the same?
$activity->save();
// Save jobconf if necessary
$hash = md5(serialize($jc->content));
if ($existingid = JobConfiguration::where('hash', $hash)->pluck('_id')) {
//[qq]
$jcid = $existingid;
// Don't save, it already exists.
} else {
$jc->format = $batch->format;
$jc->domain = $batch->domain;
$jc->project = $batch->project;
$jc->user_id = $batch->user_id;
$jc->softwareAgent_id = $activity->softwareAgent_id;
$jc->hash = $hash;
$jc->activity_id = $activity->_id;
$jc->save();
$jcid = $jc->_id;
}
// Software Agent ID which will perform the job
// TODO: Fix this -- choose one of the possibly many platforms in $jcco
// at the moment we are picking the first arbitrarily
$job_sw_agent = $jcco['platform'][0];
// Publish jobs
$j = new Job();
$j->format = $batch->format;
$j->project = $batch->project;
$j->domain = $batch->domain;
$j->user_id = $batch->user_id;
$j->type = "job";
$j->batch_id = $batch->_id;
$j->jobConf_id = $jcid;
$j->softwareAgent_id = $job_sw_agent;
$j->activity_id = $activity->_id;
$j->iamemptyjob = "yes";
$extraInfoBatch = array();
$extraInfoBatch["batchColumnsNewTemplate"] = $batchColumnsNewTemplate;
$extraInfoBatch["batchColumnsExtraChosenTemplate"] = $batchColumnsExtraChosenTemplate;
$extraInfoBatch["associationsTemplBatch"] = $associationsTemplBatch;
$extraInfoBatch["ownTemplate"] = $ownTemplate;
$j->extraInfoBatch = $extraInfoBatch;
$j->save();
//convert to publish later
$j->publish($ordersandbox == 'sandbox' ? true : false);
$jobs[] = $j;
$successmessage = "Created job with jobConf :-)";
// $platform = App::make($job_sw_agent);
$platform = App::make("cf2");
// $platform = App::make("DrDetectiveGamingPlatform"); //TODOJORAN
$platform->refreshJob($j->_id);
Session::flash('flashSuccess', $successmessage);
return Redirect::to("jobs");
} catch (Exception $e) {
// Undo creation and delete jobs
// if(isset($jobs))
// foreach($jobs as $j){
// if(isset($j->platformJobId))
// $j->undoCreation($j->platformJobId);
// $j->forceDelete();
// }
//delete activity
if (isset($j)) {
$j->forceDelete();
}
if (isset($jc)) {
$jc->forceDelete();
}
if ($activity) {
$activity->forceDelete();
}
throw $e;
//for debugging
Session::flash('flashError', $e->getMessage());
return Redirect::to("jobs2/submit");
}
}
/**
* Create job entity
* Note: the job does not get a platformJobId because this is unknown
*/
public function createJob($config, $activity, $batch, $settings)
{
$entity = new Job();
$entity->_id = $entity->_id;
$entity->batch_id = $batch->_id;
$entity->project = $settings['project'];
$entity->documentType = $settings['documentType'];
$entity->resultType = $settings['resultType'];
$entity->type = "job";
$entity->completion = 1;
$entity->expectedWorkerUnitsCount = 450;
$entity->finishedAt = new MongoDate();
$entity->jobConf_id = $config;
$entity->platformJobId = $settings['filename'];
$entity->projectedCost = 12.0;
$entity->realCost = 11.97;
$entity->runningTimeInSeconds = 190714;
$entity->softwareAgent_id = $settings['platform'];
$entity->startedAt = new MongoDate();
$entity->status = "imported";
$entity->template = "imported";
$entity->activity_id = $activity;
$entity->save();
array_push($this->status['success'], "Job created (" . $entity->_id . ")");
return $entity;
}
