/**
*
* @param array $input
* @return result status structure containing
* 'status' 'ok' or 'error'
* 'message' 'A status message'
* 'batch' the batch or null if error
*/
public function store(array $input)
{
try {
$this->createBatchCreatorSoftwareAgent();
} catch (Exception $e) {
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "batchcreator";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
try {
$entity = new Entity();
$entity->title = $input['batch_title'];
$entity->format = $input['format'];
$entity->domain = $input['domain'];
$entity->documentType = 'batch';
$entity->softwareAgent_id = 'batchcreator';
$entity->parents = $input['units'];
$entity->content = $input['batch_description'];
$entity->hash = md5(serialize($entity->parents));
$entity->activity_id = $activity->_id;
$entity->save();
Queue::push('Queues\\UpdateUnits', $input['units']);
return ['status' => 'ok', 'message' => $input['batch_title'] . " batch was successfully created. (URI: {$entity->_id})", 'batch' => $entity];
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
}
public function OldcreateWorkerunitsAndCrowdAgents($mappedWorkerunitsWithUnits, $job_id, $taskType = "FactSpan")
{
$status = array();
$index = 0;
try {
$activity = new Activity();
$activity->softwareAgent_id = "cf";
$activity->save();
} catch (Exception $e) {
$activity->forceDelete();
$status['error'][$index]['activity'] = $e->getMessage();
}
foreach ($mappedWorkerunitsWithUnits as $mappedWorkerunitsWithUnit) {
$index++;
$crowdagent = CrowdAgent::where('platformAgentId', $mappedWorkerunitsWithUnit['_worker_id'])->where('softwareAgent_id', 'cf')->first();
if (!$crowdagent) {
try {
$crowdagent = new CrowdAgent();
$crowdagent->_id = "crowdagent/cf/" . $mappedWorkerunitsWithUnit['_worker_id'];
$crowdagent->softwareAgent_id = 'cf';
$crowdagent->platformAgentId = (int) $mappedWorkerunitsWithUnit['_worker_id'];
$crowdagent->country = $mappedWorkerunitsWithUnit['_country'];
$crowdagent->region = $mappedWorkerunitsWithUnit['_region'];
$crowdagent->city = $mappedWorkerunitsWithUnit['_city'];
$crowdagent->cfWorkerTrust = (double) $mappedWorkerunitsWithUnit['_trust'];
$crowdagent->save();
} catch (Exception $e) {
$status['error'][$index]['crowdagent'] = $e->getMessage();
// continue;
}
}
if (!Entity::where('softwareAgent_id', 'cf')->where('platformWorkerunitId', $mappedWorkerunitsWithUnit['_id'])->first()) {
$entity = new Entity();
$entity->format = "text";
$entity->domain = "medical";
$entity->documentType = "workerunit";
$entity->job_id = $job_id;
$entity->activity_id = $activity->_id;
$entity->crowdAgent_id = $crowdagent->_id;
$entity->softwareAgent_id = "cf";
$entity->unit_id = $mappedWorkerunitsWithUnit['unit']['_id'];
$entity->platformWorkerunitId = (int) $mappedWorkerunitsWithUnit['_id'];
$entity->cfChannel = $mappedWorkerunitsWithUnit['_channel'];
$entity->acceptTime = new MongoDate(strtotime($mappedWorkerunitsWithUnit['_started_at']));
$entity->submitTime = new MongoDate(strtotime($mappedWorkerunitsWithUnit['_created_at']));
$entity->cfTrust = (double) $mappedWorkerunitsWithUnit['_trust'];
if ($taskType == "FactSpan") {
$entity->content = ["confirmfirstfactor" => $mappedWorkerunitsWithUnit['confirmfirstfactor'], "confirmsecondfactor" => $mappedWorkerunitsWithUnit['confirmsecondfactor'], "firstfactor" => $mappedWorkerunitsWithUnit['firstfactor'], "secondfactor" => $mappedWorkerunitsWithUnit['secondfactor'], "saveselectionids1" => $mappedWorkerunitsWithUnit['saveselectionids1'], "saveselectionids2" => $mappedWorkerunitsWithUnit['saveselectionids2'], "confirmids1" => $mappedWorkerunitsWithUnit['confirmids1'], "confirmids2" => $mappedWorkerunitsWithUnit['confirmids2'], "sentencefirstfactor" => $mappedWorkerunitsWithUnit['sentencefirstfactor'], "sentencesecondfactor" => $mappedWorkerunitsWithUnit['sentencesecondfactor']];
} elseif ($taskType == "RelEx") {
$entity->content = ["step_1_select_the_valid_relations" => $mappedWorkerunitsWithUnit['step_1_select_the_valid_relations'], "step_2a_copy__paste_only_the_words_from_the_sentence_that_express_the_relation_you_selected_in_step1" => $mappedWorkerunitsWithUnit['step_2a_copy__paste_only_the_words_from_the_sentence_that_express_the_relation_you_selected_in_step1'], "step_2b_if_you_selected_none_in_step_1_explain_why" => $mappedWorkerunitsWithUnit['step_2b_if_you_selected_none_in_step_1_explain_why']];
} elseif ($taskType == "RelDir") {
$entity->content = ["direction" => $mappedWorkerunitsWithUnit['direction']];
}
try {
$entity->save();
} catch (Exception $e) {
$status['error'][$index]['entity'] = $e->getMessage();
}
}
}
return $status;
}
public function getProcesscrowdgames()
{
$gameJobs = Job::where('softwareAgent_id', 'DrDetectiveGamingPlatform')->get();
$activity = new Activity();
$activity->softwareAgent_id = 'DrDetectiveGamingPlatform';
$activity->save();
foreach ($gameJobs as $job) {
// $annotations = Entity::where('jobParents', $job['_id'])->get();
// Create one annotation vector for each image on the game
$images = Entity::where('jobParents', $job['_id'])->distinct('content.task_data')->get();
$annotationsSummary = [];
foreach ($images as $image) {
$imageName = $image[0];
// unpack data
$annotations = Entity::where('jobParents', $job['_id'])->where('content.task_data', $imageName)->get();
$annotations = $annotations->toArray();
// Create an array with all coordinates given for target image.
$coordinates = array_column(array_column(array_column($annotations, 'content'), 'response'), 'Coordinates');
$allCoordinates = [];
foreach ($coordinates as $coords) {
// Flatten to array of coords.
foreach ($coords as $c) {
$allCoordinates[] = $c;
}
}
$aggCoords = static::aggregateCoordinates($allCoordinates);
$annotationsSummary[] = ['image' => $imageName, 'aggregateCoordinates' => $aggCoords];
}
// process annotations for this job into an annotation vector...
$e = new Entity();
$e->jobParents = [$job['_id']];
$e->annotationVector = $annotationsSummary;
$e->documentType = 'annotationVector';
$e->activity_id = $activity->_id;
$e->softwareAgent_id = $job->softwareAgent_id;
$e->project = $job->project;
$e->user_id = $job->user_id;
$e->save();
}
return 'OK -- may need adjustments...';
}
public function store($format, $domain, $documentType, $parameters, $noOfVideos)
{
//fastcgi_finish_request();
$listOfVideoIdentifiers = array();
$this->listRecords($parameters, $noOfVideos, $listOfVideoIdentifiers);
// dd("done");
$status = array();
try {
$this->createOpenimagesVideoGetterSoftwareAgent();
} catch (Exception $e) {
$status['error']['OnlineData'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "openimagesgetter";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$status['error']['OnlineData'] = $e->getMessage();
$activity->forceDelete();
return $status;
}
$count["count"] = 0;
foreach ($listOfVideoIdentifiers as $video) {
$title = $video;
try {
$entity = new Entity();
$entity->_id = $entity->_id;
$entity->title = strtolower($title);
$entity->domain = $domain;
$entity->format = $format;
$entity->documentType = $documentType;
$entity->source = "openimages";
$videoMetadata = $this->getRecord($video, $parameters["metadataPrefix"]);
$entity->content = $videoMetadata["content"];
$parents = array();
$entity->parents = $parents;
$entity->tags = array("unit");
$entity->segments = $count;
$entity->keyframes = $count;
$entity->hash = md5(serialize([$entity->content]));
$entity->activity_id = $activity->_id;
$entity->save();
Queue::push('Queues\\UpdateUnits', [$entity->_id]);
$status['success'][$title] = $title . " was successfully uploaded. (URI: {$entity->_id})";
if (isset($status['success'])) {
$this->storeVideoDescription($entity);
// dd($this->storeVideoDescription($entity));
}
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
}
$status["recno"] = count($listOfVideoIdentifiers);
return $status;
}
/**
* update units to new data structure
*/
public function postUpdatedb()
{
$searchComponent = new MediaSearchComponent();
// amount of units to index per iteration
$batchsize = 50;
$from = Input::get('next');
$unitCount = Entity::whereIn('tags', ['unit'])->count();
// reset index on start
if ($from == 0) {
$searchComponent->clear();
}
// reduce last batch to remaining units
if ($from + $batchsize > $unitCount) {
$batchsize = $unitCount - $from;
}
// all units in this range
$units = Entity::distinct('_id')->where('tags', ['unit'])->skip($from)->take($batchsize)->get();
// get list of existing projects
$projects = ProjectHandler::listGroups();
// for each unit get the keys and check if the project exists
$allKeys = [];
for ($i = $from; $i < $from + $batchsize; $i++) {
// get data of unit
$unit = Entity::where('_id', $units[$i][0])->first();
switch ($unit['documentType']) {
case 'annotatedmetadatadescription':
$unit['project'] = 'soundandvision';
break;
case 'biographynet-sentence':
$unit['project'] = 'biographynet';
break;
case 'drawing':
$unit['project'] = 'rijksmuseum';
break;
case 'enrichedvideo':
$unit['project'] = 'soundandvision';
break;
case 'enrichedvideov2':
$unit['project'] = 'soundandvision';
break;
case 'enrichedvideov3':
$unit['project'] = 'soundandvision';
break;
case 'fullvideo':
$unit['project'] = 'soundandvision';
break;
case 'metadatadescription':
$unit['project'] = 'soundandvision';
break;
case 'metadatadescription-event':
$unit['project'] = 'soundandvision';
break;
case 'painting':
$unit['project'] = 'rijksmuseum';
break;
case 'relex':
$unit['project'] = 'ibmrelex';
break;
case 'relex-sentence':
$unit['project'] = 'ibmrelex';
break;
case 'relex-structured-sentence':
$unit['project'] = 'ibmrelex';
break;
case 'termpairs-sentence':
$unit['project'] = 'ibmdisdis';
break;
}
// add the project if it doesnt exist yet
if (!in_array($unit['project'], $projects)) {
ProjectHandler::createGroup($unit['project']);
// add the project to the temporary list
array_push($projects, $unit['project']);
}
// add the user to the project if it has no access yet
if (!ProjectHandler::inGroup($unit['user_id'], $unit['project'])) {
$user = UserAgent::find($unit['user_id']);
ProjectHandler::grantUser($user, $unit['project'], Roles::PROJECT_MEMBER);
}
$id = explode('/', $unit['_id']);
if (sizeof($id) == 5) {
$entity = new Entity();
// copy properties
$entity['documentType'] = $unit['documentType'];
$entity['test'] = 'new';
$entity->_id = 'entity/' . $entity->documentType . '/' . $id[4];
$entity->save();
}
}
return ['log' => 'test', 'next' => $from + $batchsize, 'last' => $unitCount];
}
public function postFeatures()
{
$return = array('status' => 'ok');
$input = Input::get();
$domain = $input[1];
$type = $input[2];
// CREATE ACTIVITY FOR BATCH
$activity = new Activity();
$activity->label = "Images posted for processing.";
$activity->softwareAgent_id = 'imagegetter';
$activity->save();
// LOOP THROUGH IMAGES CREATE ENTITIES WITH ACTIVITY-ID FOR NEW IMAGES
$url_ids = "";
foreach ($input[0] as $img) {
\Log::debug(json_encode($img));
try {
$parse = parse_url($img['url']);
//$source = $parse['host'];
// Save images as parent
$image = new Entity();
$image->domain = $domain;
$image->format = "image";
$content = $image->content;
$content['url'] = $img['url'];
$content['title'] = $img['title'];
$content['height'] = $img['height'];
$content['width'] = $img['width'];
$content['description'] = $img['description'];
$content['author'] = $img['author'];
$image->content = $content;
$image->documentType = $type;
$image->source = "Rijksmuseum";
$image->tags = ['unit'];
$image->activity_id = $activity->_id;
$image->softwareAgent_id = "imagegetter";
// Take last part of URL as image title
$temp = explode('/', $img['url']);
//$image->title = end($temp);
// CHECK WHETHER URL EXISTS ALREADY
$hash = md5(serialize($image->content));
if ($existingid = Entity::where('hash', $hash)->pluck('_id')) {
$imageid = $existingid;
} else {
$image->hash = $hash;
$image->activity_id = $activity->_id;
\Log::debug(json_encode($image->toArray()));
$image->save();
$existingid = $image->_id;
}
$url_ids .= "{$img['url']} {$existingid} ";
} catch (Exception $e) {
//delete image
if (isset($image)) {
$image->forceDelete();
}
//delete activity
if (isset($activity)) {
$activity->forceDelete();
}
//Session::flash('flashError', $e->getMessage());
$return['error'] = $e->getMessage();
$return['status'] = 'bad';
\Log::debug($e->getMessage());
return $return;
}
// RUN PYTHON SCRIPT THAT CALLS APIs TO ADD FEATURES TO IMAGE
}
//return $url_ids;
try {
//$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getRijks.py " . $domain . " " . $type . " " . 4 . " " . "vogel";
$command = "/usr/bin/python2.7 " . base_path() . "/app/lib/getAPIS/getMany.py " . $domain . " " . $type . " " . Auth::user()->email . " " . $url_ids;
//$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getMany.py art painting http://lh3.ggpht.com/Q1GZTdmwa8iTLgdbu5uAgzovmLbb7lsYhG-QgVcoN8A-WJtIsNUo4-VyTMd9iKHLp-XNm812WyUaSgQdHdjQjDioJQI=s0 999";
//return $command;
\Log::debug("Running {$command}");
exec($command, $output, $error);
$return['oo'] = $output;
$return['ee'] = $error;
//$return['a'] = $a;
//throw $e; // for debugging.
//return $error;
} catch (Exception $e) {
//throw $e; // for debugging.
\Log::debug("ERROR: " . $e->getMessage());
$return['error'] = $e->getMessage();
$return['status'] = 'bad';
}
return $this->returnJson($return);
}
public function storeVideoSegments($parentEntity, $videoSegmenting)
{
$tempEntityID = null;
$status = array();
try {
$this->createVideoSegmentingSoftwareAgent();
} catch (Exception $e) {
$status['error']['videosegmenting'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "videosegmenting";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error'][$title] = $e->getMessage();
return $status;
}
for ($i = 0; $i < sizeof($videoSegmenting); $i++) {
$videoSegmentName = explode("/", $videoSegmenting[$i]["storage_url"]);
$title = $videoSegmentName[sizeof($videoSegmentName) - 1];
try {
$entity = new Entity();
$entity->_id = $tempEntityID;
$entity->title = strtolower($title);
$entity->domain = $parentEntity->domain;
$entity->format = "video";
$entity->documentType = "videosegment";
$entity->parents = array($parentEntity->_id);
$entity->source = $parentEntity->source;
$entity->content = $videoSegmenting[$i];
//unset($relexStructuredSentenceKeyVal['properties']);
$entity->hash = md5(serialize($videoSegmenting[$i]));
$entity->activity_id = $activity->_id;
$entity->save();
$status['success'][$title] = $title . " was successfully processed into a video segment. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
$tempEntityID = $entity->_id;
}
$status['success']['noEntitiesCreated'] = sizeof($videoSegmenting);
//dd($status);
return $status;
}
public function anyTest()
{
try {
if (!($data = Input::get('data'))) {
if (!($data = \Request::getContent())) {
return false;
}
}
if (empty($data)) {
return false;
}
$data = json_decode($data, true);
$data['softwareAgent_id'] = strtolower($data['softwareAgent_id']);
try {
$this->createPostSoftwareAgent($data);
} catch (Exception $e) {
return serialize([$e->getMessage()]);
}
try {
$activity = new Activity();
$activity->softwareAgent_id = $data['softwareAgent_id'];
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
return serialize([$e->getMessage()]);
}
$entity = new Entity();
$entity->format = 'image';
$entity->domain = $data['domain'];
$entity->tags = $data['tags'];
$entity->documentType = $data['documentType'];
$entity->softwareAgent_configuration = $data['softwareAgent_configuration'];
if (isset($data['parents'])) {
$entity->parents = $data['parents'];
}
$entity->content = $data['content'];
if (isset($data['threshold'])) {
$entity->threshold = $data['threshold'];
}
if (isset($data['relevantFeatures'])) {
$entity->recognizedFeatures = $data['relevantFeatures'];
}
if (isset($data['hash'])) {
$entity->hash = $data['hash'];
} else {
$entity->hash = md5(serialize($data['content']));
}
$entity->activity_id = $activity->_id;
if (Entity::where('hash', $entity->hash)->first()) {
//dd('asdasd');
} else {
$entity->save();
}
return Response::json($entity);
} catch (Exception $e) {
dd($e->getMessage());
}
}
/**
* Store a new file to the database. Construct all entity information for such file.
*
* @param $fileFormat
* @param $domain
* @param $documentType
* @param $project The name of the Project who owns the file data.
* @param $domainCreate
* @param $documentCreate
* @param $files
*/
public function store($fileFormat, $domain, $documentType, $project, $domainCreate, $documentCreate, $files)
{
$format = $this->getType($fileFormat);
$validatedFiles = $this->performValidation($files, $format);
$newDomain = false;
$newDocType = false;
if ($domain == 'domain_type_other') {
// Add new domain to DB
$domain = $domainCreate;
$domain = str_replace(' ', '', $domain);
$domain = strtolower($domain);
$domain = 'domain_type_' . $domain;
$newDomain = true;
}
if ($documentType == 'document_type_other') {
// Add new doc_type to DB
$documentType = $documentCreate;
$newDocType;
}
if ($newDomain || $newDocType) {
if ($newDomain) {
// newDomain and new DocType
$domainName = $domainCreate;
$upDomains = $this->softwareComponent->domains;
$upDomains[$domain] = ["name" => $domainName, "file_formats" => [$fileFormat], "document_types" => [$documentType]];
$this->softwareComponent->domains = $upDomains;
} else {
if ($newDocType) {
// Only docType is new -- domain already existed...
$docTypes = $this->softwareComponent->domains[$domain]["document_types"];
array_push($docTypes, $documentType);
$this->softwareComponent->domains[$domain]["document_types"] = $docTypes;
}
}
$this->softwareComponent->save();
}
$domain = str_replace("domain_type_", "", $domain);
$documentType = str_replace("document_type_", "", $documentType);
$status = [];
try {
$activity = new Activity();
$activity->softwareAgent_id = $this->softwareComponent->_id;
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error'] = $e->getMessage();
return $status;
}
$files = $validatedFiles['passed'];
foreach ($files as $file) {
$title = $file->getClientOriginalName();
try {
$entity = new Entity();
$entity->_id = $entity->_id;
$entity->activity_id = $activity->_id;
$entity->softwareAgent_id = $this->softwareComponent->_id;
$entity->project = $project;
$entity->title = strtolower($title);
$entity->domain = $domain;
$entity->format = "text";
$entity->documentType = $documentType;
$entity->content = File::get($file->getRealPath());
$entity->hash = md5(serialize([$entity->content]));
$entity->tags = ["unit"];
$entity->save();
$status['success'][$title] = $title . " was successfully uploaded. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
}
$files = $validatedFiles['failed'];
foreach ($files as $file) {
$title = $file->getClientOriginalName();
$status['error'][$title] = 'Validation failed';
}
return $status;
}
