public static function boot()
{
parent::boot();
static::creating(function ($jobconf) {
// IFEXISTS CHECK IS NOT HERE.
try {
$c = $jobconf->content;
if (isset($c['reward'])) {
$c['reward'] = (double) $c['reward'];
}
if (isset($c['hitLifetimeInMinutes'])) {
$c['hitLifetimeInMinutes'] = intval($c['hitLifetimeInMinutes']);
}
if (isset($c['autoApprovalDelayInMinutes'])) {
$c['autoApprovalDelayInMinutes'] = intval($c['autoApprovalDelayInMinutes']);
}
if (isset($c['expirationInMinutes'])) {
$c['expirationInMinutes'] = intval($c['expirationInMinutes']);
}
if (isset($c['workerunitsPerUnit'])) {
$c['workerunitsPerUnit'] = intval($c['workerunitsPerUnit']);
}
if (isset($c['unitsPerTask'])) {
$c['unitsPerTask'] = intval($c['unitsPerTask']);
if ($c['unitsPerTask'] == 0) {
$c['unitsPerTask'] = 1;
}
}
$jobconf->content = $c;
} catch (Exception $e) {
if ($jobconf) {
$jobconf->forceDelete();
}
throw new Exception('Error saving JobConfiguration.');
}
if (empty($jobconf->activity_id)) {
try {
$activity = new Activity();
$activity->label = "JobConfiguration is saved.";
$activity->softwareAgent_id = 'jobcreator';
$activity->save();
$jobconf->activity_id = $activity->_id;
Log::debug("Saved JobConfiguration with activity {$jobconf->activity_id}.");
} catch (Exception $e) {
if ($activity) {
$activity->forceDelete();
}
if ($jobconf) {
$jobconf->forceDelete();
}
throw new Exception('Error saving activity for JobConfiguration.');
}
}
});
}
/**
*
* @param array $input
* @return result status structure containing
* 'status' 'ok' or 'error'
* 'message' 'A status message'
* 'batch' the batch or null if error
*/
public function store(array $input)
{
try {
$this->createBatchCreatorSoftwareAgent();
} catch (Exception $e) {
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "batchcreator";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
try {
$entity = new Entity();
$entity->title = $input['batch_title'];
$entity->format = $input['format'];
$entity->domain = $input['domain'];
$entity->documentType = 'batch';
$entity->softwareAgent_id = 'batchcreator';
$entity->parents = $input['units'];
$entity->content = $input['batch_description'];
$entity->hash = md5(serialize($entity->parents));
$entity->activity_id = $activity->_id;
$entity->save();
Queue::push('Queues\\UpdateUnits', $input['units']);
return ['status' => 'ok', 'message' => $input['batch_title'] . " batch was successfully created. (URI: {$entity->_id})", 'batch' => $entity];
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
}
public static function boot()
{
parent::boot();
static::saving(function ($questiontemplate) {
if (empty($questiontemplate->activity_id)) {
try {
$activity = new Activity();
$activity->label = "Questiontemplate is saved.";
$activity->softwareAgent_id = 'templatebuilder';
$activity->save();
$questiontemplate->activity_id = $activity->_id;
Log::debug("Saving QuestionTemplate {$questiontemplate->_id} with activity {$questiontemplate->activity_id}.");
} catch (Exception $e) {
if ($activity) {
$activity->forceDelete();
}
if ($questiontemplate) {
$questiontemplate->forceDelete();
}
throw new Exception('Error saving activity for QuestionTemplate.');
}
}
});
}
public function OldcreateWorkerunitsAndCrowdAgents($mappedWorkerunitsWithUnits, $job_id, $taskType = "FactSpan")
{
$status = array();
$index = 0;
try {
$activity = new Activity();
$activity->softwareAgent_id = "cf";
$activity->save();
} catch (Exception $e) {
$activity->forceDelete();
$status['error'][$index]['activity'] = $e->getMessage();
}
foreach ($mappedWorkerunitsWithUnits as $mappedWorkerunitsWithUnit) {
$index++;
$crowdagent = CrowdAgent::where('platformAgentId', $mappedWorkerunitsWithUnit['_worker_id'])->where('softwareAgent_id', 'cf')->first();
if (!$crowdagent) {
try {
$crowdagent = new CrowdAgent();
$crowdagent->_id = "crowdagent/cf/" . $mappedWorkerunitsWithUnit['_worker_id'];
$crowdagent->softwareAgent_id = 'cf';
$crowdagent->platformAgentId = (int) $mappedWorkerunitsWithUnit['_worker_id'];
$crowdagent->country = $mappedWorkerunitsWithUnit['_country'];
$crowdagent->region = $mappedWorkerunitsWithUnit['_region'];
$crowdagent->city = $mappedWorkerunitsWithUnit['_city'];
$crowdagent->cfWorkerTrust = (double) $mappedWorkerunitsWithUnit['_trust'];
$crowdagent->save();
} catch (Exception $e) {
$status['error'][$index]['crowdagent'] = $e->getMessage();
// continue;
}
}
if (!Entity::where('softwareAgent_id', 'cf')->where('platformWorkerunitId', $mappedWorkerunitsWithUnit['_id'])->first()) {
$entity = new Entity();
$entity->format = "text";
$entity->domain = "medical";
$entity->documentType = "workerunit";
$entity->job_id = $job_id;
$entity->activity_id = $activity->_id;
$entity->crowdAgent_id = $crowdagent->_id;
$entity->softwareAgent_id = "cf";
$entity->unit_id = $mappedWorkerunitsWithUnit['unit']['_id'];
$entity->platformWorkerunitId = (int) $mappedWorkerunitsWithUnit['_id'];
$entity->cfChannel = $mappedWorkerunitsWithUnit['_channel'];
$entity->acceptTime = new MongoDate(strtotime($mappedWorkerunitsWithUnit['_started_at']));
$entity->submitTime = new MongoDate(strtotime($mappedWorkerunitsWithUnit['_created_at']));
$entity->cfTrust = (double) $mappedWorkerunitsWithUnit['_trust'];
if ($taskType == "FactSpan") {
$entity->content = ["confirmfirstfactor" => $mappedWorkerunitsWithUnit['confirmfirstfactor'], "confirmsecondfactor" => $mappedWorkerunitsWithUnit['confirmsecondfactor'], "firstfactor" => $mappedWorkerunitsWithUnit['firstfactor'], "secondfactor" => $mappedWorkerunitsWithUnit['secondfactor'], "saveselectionids1" => $mappedWorkerunitsWithUnit['saveselectionids1'], "saveselectionids2" => $mappedWorkerunitsWithUnit['saveselectionids2'], "confirmids1" => $mappedWorkerunitsWithUnit['confirmids1'], "confirmids2" => $mappedWorkerunitsWithUnit['confirmids2'], "sentencefirstfactor" => $mappedWorkerunitsWithUnit['sentencefirstfactor'], "sentencesecondfactor" => $mappedWorkerunitsWithUnit['sentencesecondfactor']];
} elseif ($taskType == "RelEx") {
$entity->content = ["step_1_select_the_valid_relations" => $mappedWorkerunitsWithUnit['step_1_select_the_valid_relations'], "step_2a_copy__paste_only_the_words_from_the_sentence_that_express_the_relation_you_selected_in_step1" => $mappedWorkerunitsWithUnit['step_2a_copy__paste_only_the_words_from_the_sentence_that_express_the_relation_you_selected_in_step1'], "step_2b_if_you_selected_none_in_step_1_explain_why" => $mappedWorkerunitsWithUnit['step_2b_if_you_selected_none_in_step_1_explain_why']];
} elseif ($taskType == "RelDir") {
$entity->content = ["direction" => $mappedWorkerunitsWithUnit['direction']];
}
try {
$entity->save();
} catch (Exception $e) {
$status['error'][$index]['entity'] = $e->getMessage();
}
}
}
return $status;
}
public function postSubmitFinal($ordersandbox = 'order')
{
$jc = unserialize(Session::get('jobconf'));
$template = Session::get('template');
$batch = unserialize(Session::get('batch'));
$questiontemplateid = Session::get('questiontemplateid');
$jobs = array();
if (!$jc->validate()) {
$msg = '';
foreach ($jc->getErrors()->all() as $message) {
$msg .= "<li>{$message}</li>";
}
Session::flash('flashError', "<ul>{$msg}</ul>");
return Redirect::to("jobs/submit");
}
try {
// Save activity
$activity = new MongoDB\Activity();
$activity->label = "Job is uploaded to crowdsourcing platform.";
$activity->softwareAgent_id = 'jobcreator';
// JOB softwareAgent_id = $platform. Does this need to be the same?
$activity->save();
// Save jobconf if necessary
$hash = md5(serialize($jc->content));
if ($existingid = JobConfiguration::where('hash', $hash)->pluck('_id')) {
$jcid = $existingid;
} else {
$jc->format = $batch->format;
$jc->domain = $batch->domain;
$jc->hash = $hash;
$jc->activity_id = $activity->_id;
$jc->save();
$jcid = $jc->_id;
}
// Publish jobs
foreach ($jc->content['platform'] as $platformstring) {
$j = new Job();
$j->format = $batch->format;
$j->domain = $batch->domain;
$j->type = explode('/', $template)[1];
$j->template = $template;
// TODO: remove
$j->batch_id = $batch->_id;
$j->questionTemplate_id = $questiontemplateid;
$j->jobConf_id = $jcid;
$j->softwareAgent_id = $platformstring;
$j->activity_id = $activity->_id;
$j->publish($ordersandbox == 'sandbox' ? true : false);
$jobs[] = $j;
}
// Success.
//Session::flash('flashSuccess', "Created " . ($ordersandbox == 'sandbox' ? 'but didn\'t order' : 'and ordered') . " job(s) on " .
// strtoupper(implode(', ', $jc->content['platform'])) . '.');
$successmessage = "Created job" . (count($jc->content['platform']) > 1 ? 's' : '') . " on " . strtoupper(implode(', ', $jc->content['platform'])) . '. Order it by pressing the button under \'Actions\'. Demo jobs are published on the sandbox or internal channels only.';
// TODO: this only takes the first job of potentially two
if (!empty($jobs[0]->url)) {
$successmessage .= ". After that, you can view it <a href='{$jobs[0]->url}' target='blank'>here</a>.";
}
Session::flash('flashSuccess', $successmessage);
return Redirect::to("jobs/");
//(Auth::user()->role == 'demo' ? '. Because this is a demo account, you can not order it. Please take a look at our finished jobs!' : '. Click on \'actions\' on the job to order it.')
} catch (Exception $e) {
// Undo creation and delete jobs
if (isset($jobs)) {
foreach ($jobs as $j) {
if (isset($j->platformJobId)) {
$j->undoCreation($j->platformJobId);
}
$j->forceDelete();
}
}
//delete activity
if ($activity) {
$activity->forceDelete();
}
throw $e;
//for debugging
Session::flash('flashError', $e->getMessage());
return Redirect::to("jobs/submit");
}
}
public function store($format, $domain, $documentType, $parameters, $noOfVideos)
{
//fastcgi_finish_request();
$listOfVideoIdentifiers = array();
$this->listRecords($parameters, $noOfVideos, $listOfVideoIdentifiers);
// dd("done");
$status = array();
try {
$this->createOpenimagesVideoGetterSoftwareAgent();
} catch (Exception $e) {
$status['error']['OnlineData'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "openimagesgetter";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$status['error']['OnlineData'] = $e->getMessage();
$activity->forceDelete();
return $status;
}
$count["count"] = 0;
foreach ($listOfVideoIdentifiers as $video) {
$title = $video;
try {
$entity = new Entity();
$entity->_id = $entity->_id;
$entity->title = strtolower($title);
$entity->domain = $domain;
$entity->format = $format;
$entity->documentType = $documentType;
$entity->source = "openimages";
$videoMetadata = $this->getRecord($video, $parameters["metadataPrefix"]);
$entity->content = $videoMetadata["content"];
$parents = array();
$entity->parents = $parents;
$entity->tags = array("unit");
$entity->segments = $count;
$entity->keyframes = $count;
$entity->hash = md5(serialize([$entity->content]));
$entity->activity_id = $activity->_id;
$entity->save();
Queue::push('Queues\\UpdateUnits', [$entity->_id]);
$status['success'][$title] = $title . " was successfully uploaded. (URI: {$entity->_id})";
if (isset($status['success'])) {
$this->storeVideoDescription($entity);
// dd($this->storeVideoDescription($entity));
}
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
}
$status["recno"] = count($listOfVideoIdentifiers);
return $status;
}
public function postFeatures()
{
$return = array('status' => 'ok');
$input = Input::get();
$domain = $input[1];
$type = $input[2];
// CREATE ACTIVITY FOR BATCH
$activity = new Activity();
$activity->label = "Images posted for processing.";
$activity->softwareAgent_id = 'imagegetter';
$activity->save();
// LOOP THROUGH IMAGES CREATE ENTITIES WITH ACTIVITY-ID FOR NEW IMAGES
$url_ids = "";
foreach ($input[0] as $img) {
\Log::debug(json_encode($img));
try {
$parse = parse_url($img['url']);
//$source = $parse['host'];
// Save images as parent
$image = new Entity();
$image->domain = $domain;
$image->format = "image";
$content = $image->content;
$content['url'] = $img['url'];
$content['title'] = $img['title'];
$content['height'] = $img['height'];
$content['width'] = $img['width'];
$content['description'] = $img['description'];
$content['author'] = $img['author'];
$image->content = $content;
$image->documentType = $type;
$image->source = "Rijksmuseum";
$image->tags = ['unit'];
$image->activity_id = $activity->_id;
$image->softwareAgent_id = "imagegetter";
// Take last part of URL as image title
$temp = explode('/', $img['url']);
//$image->title = end($temp);
// CHECK WHETHER URL EXISTS ALREADY
$hash = md5(serialize($image->content));
if ($existingid = Entity::where('hash', $hash)->pluck('_id')) {
$imageid = $existingid;
} else {
$image->hash = $hash;
$image->activity_id = $activity->_id;
\Log::debug(json_encode($image->toArray()));
$image->save();
$existingid = $image->_id;
}
$url_ids .= "{$img['url']} {$existingid} ";
} catch (Exception $e) {
//delete image
if (isset($image)) {
$image->forceDelete();
}
//delete activity
if (isset($activity)) {
$activity->forceDelete();
}
//Session::flash('flashError', $e->getMessage());
$return['error'] = $e->getMessage();
$return['status'] = 'bad';
\Log::debug($e->getMessage());
return $return;
}
// RUN PYTHON SCRIPT THAT CALLS APIs TO ADD FEATURES TO IMAGE
}
//return $url_ids;
try {
//$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getRijks.py " . $domain . " " . $type . " " . 4 . " " . "vogel";
$command = "/usr/bin/python2.7 " . base_path() . "/app/lib/getAPIS/getMany.py " . $domain . " " . $type . " " . Auth::user()->email . " " . $url_ids;
//$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getMany.py art painting http://lh3.ggpht.com/Q1GZTdmwa8iTLgdbu5uAgzovmLbb7lsYhG-QgVcoN8A-WJtIsNUo4-VyTMd9iKHLp-XNm812WyUaSgQdHdjQjDioJQI=s0 999";
//return $command;
\Log::debug("Running {$command}");
exec($command, $output, $error);
$return['oo'] = $output;
$return['ee'] = $error;
//$return['a'] = $a;
//throw $e; // for debugging.
//return $error;
} catch (Exception $e) {
//throw $e; // for debugging.
\Log::debug("ERROR: " . $e->getMessage());
$return['error'] = $e->getMessage();
$return['status'] = 'bad';
}
return $this->returnJson($return);
}
public function storeVideoSegments($parentEntity, $videoSegmenting)
{
$tempEntityID = null;
$status = array();
try {
$this->createVideoSegmentingSoftwareAgent();
} catch (Exception $e) {
$status['error']['videosegmenting'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "videosegmenting";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error'][$title] = $e->getMessage();
return $status;
}
for ($i = 0; $i < sizeof($videoSegmenting); $i++) {
$videoSegmentName = explode("/", $videoSegmenting[$i]["storage_url"]);
$title = $videoSegmentName[sizeof($videoSegmentName) - 1];
try {
$entity = new Entity();
$entity->_id = $tempEntityID;
$entity->title = strtolower($title);
$entity->domain = $parentEntity->domain;
$entity->format = "video";
$entity->documentType = "videosegment";
$entity->parents = array($parentEntity->_id);
$entity->source = $parentEntity->source;
$entity->content = $videoSegmenting[$i];
//unset($relexStructuredSentenceKeyVal['properties']);
$entity->hash = md5(serialize($videoSegmenting[$i]));
$entity->activity_id = $activity->_id;
$entity->save();
$status['success'][$title] = $title . " was successfully processed into a video segment. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
$tempEntityID = $entity->_id;
}
$status['success']['noEntitiesCreated'] = sizeof($videoSegmenting);
//dd($status);
return $status;
}
public function storeNERDApi($parentEntity, $metadataDescriptionPreprocessing)
{
// $metadataDescriptionPreprocessing = unserialize(utf8_decode(utf8_encode(serialize($metadataDescriptionPreprocessing))));
$status = array();
try {
$this->createNamedEntitiesExtractionNERDApiSoftwareAgent();
} catch (Exception $e) {
$status['error']['nerd'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "nerdextractor";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error'][$title] = $e->getMessage();
return $status;
}
$tempEntityID = null;
$title = "nerdextractor-" . $parentEntity["title"];
//dd($title);
try {
$entity = new Entity();
$entity->_id = $tempEntityID;
$entity->title = strtolower($title);
$entity->domain = $parentEntity->domain;
$entity->format = "text";
$entity->documentType = "nerdextractor";
$entity->parents = array($parentEntity->_id);
$entity->source = $parentEntity->source;
$content = array();
$content["description"] = $parentEntity->content;
foreach ($metadataDescriptionPreprocessing as $key => $value) {
$content["features"][$key] = $value;
}
$entity->content = $content;
//
//unset($twrexStructuredSentenceKeyVal['properties']);
$entity->hash = md5(serialize($entity));
$entity->activity_id = $activity->_id;
$entity->save();
$status['success'][$title] = $title . " was successfully processed into entities extraction. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
$tempEntityID = $entity->_id;
return $status;
}
public function anyTest()
{
try {
if (!($data = Input::get('data'))) {
if (!($data = \Request::getContent())) {
return false;
}
}
if (empty($data)) {
return false;
}
$data = json_decode($data, true);
$data['softwareAgent_id'] = strtolower($data['softwareAgent_id']);
try {
$this->createPostSoftwareAgent($data);
} catch (Exception $e) {
return serialize([$e->getMessage()]);
}
try {
$activity = new Activity();
$activity->softwareAgent_id = $data['softwareAgent_id'];
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
return serialize([$e->getMessage()]);
}
$entity = new Entity();
$entity->format = 'image';
$entity->domain = $data['domain'];
$entity->tags = $data['tags'];
$entity->documentType = $data['documentType'];
$entity->softwareAgent_configuration = $data['softwareAgent_configuration'];
if (isset($data['parents'])) {
$entity->parents = $data['parents'];
}
$entity->content = $data['content'];
if (isset($data['threshold'])) {
$entity->threshold = $data['threshold'];
}
if (isset($data['relevantFeatures'])) {
$entity->recognizedFeatures = $data['relevantFeatures'];
}
if (isset($data['hash'])) {
$entity->hash = $data['hash'];
} else {
$entity->hash = md5(serialize($data['content']));
}
$entity->activity_id = $activity->_id;
if (Entity::where('hash', $entity->hash)->first()) {
//dd('asdasd');
} else {
$entity->save();
}
return Response::json($entity);
} catch (Exception $e) {
dd($e->getMessage());
}
}
/**
* Store a new file to the database. Construct all entity information for such file.
*
* @param $fileFormat
* @param $domain
* @param $documentType
* @param $project The name of the Project who owns the file data.
* @param $domainCreate
* @param $documentCreate
* @param $files
*/
public function store($fileFormat, $domain, $documentType, $project, $domainCreate, $documentCreate, $files)
{
$format = $this->getType($fileFormat);
$validatedFiles = $this->performValidation($files, $format);
$newDomain = false;
$newDocType = false;
if ($domain == 'domain_type_other') {
// Add new domain to DB
$domain = $domainCreate;
$domain = str_replace(' ', '', $domain);
$domain = strtolower($domain);
$domain = 'domain_type_' . $domain;
$newDomain = true;
}
if ($documentType == 'document_type_other') {
// Add new doc_type to DB
$documentType = $documentCreate;
$newDocType;
}
if ($newDomain || $newDocType) {
if ($newDomain) {
// newDomain and new DocType
$domainName = $domainCreate;
$upDomains = $this->softwareComponent->domains;
$upDomains[$domain] = ["name" => $domainName, "file_formats" => [$fileFormat], "document_types" => [$documentType]];
$this->softwareComponent->domains = $upDomains;
} else {
if ($newDocType) {
// Only docType is new -- domain already existed...
$docTypes = $this->softwareComponent->domains[$domain]["document_types"];
array_push($docTypes, $documentType);
$this->softwareComponent->domains[$domain]["document_types"] = $docTypes;
}
}
$this->softwareComponent->save();
}
$domain = str_replace("domain_type_", "", $domain);
$documentType = str_replace("document_type_", "", $documentType);
$status = [];
try {
$activity = new Activity();
$activity->softwareAgent_id = $this->softwareComponent->_id;
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error'] = $e->getMessage();
return $status;
}
$files = $validatedFiles['passed'];
foreach ($files as $file) {
$title = $file->getClientOriginalName();
try {
$entity = new Entity();
$entity->_id = $entity->_id;
$entity->activity_id = $activity->_id;
$entity->softwareAgent_id = $this->softwareComponent->_id;
$entity->project = $project;
$entity->title = strtolower($title);
$entity->domain = $domain;
$entity->format = "text";
$entity->documentType = $documentType;
$entity->content = File::get($file->getRealPath());
$entity->hash = md5(serialize([$entity->content]));
$entity->tags = ["unit"];
$entity->save();
$status['success'][$title] = $title . " was successfully uploaded. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
}
$files = $validatedFiles['failed'];
foreach ($files as $file) {
$title = $file->getClientOriginalName();
$status['error'][$title] = 'Validation failed';
}
return $status;
}
