public static function boot()
{
parent::boot();
static::creating(function ($workerunit) {
// dd($workerunit);
// Inherit type, domain and format
if (empty($workerunit->type) or empty($workerunit->domain) or empty($workerunit->format)) {
$j = Job::where('_id', $workerunit->job_id)->first();
$workerunit->type = $j->type;
$workerunit->domain = $j->domain;
$workerunit->format = $j->format;
}
$workerunit->annotationVector = $workerunit->createAnnotationVector();
// Activity if not exists
if (empty($workerunit->activity_id)) {
try {
$activity = new Activity();
$activity->label = "Workerunit is saved.";
$activity->softwareAgent_id = $workerunit->softwareAgent_id;
$activity->save();
$workerunit->activity_id = $activity->_id;
Log::debug("Saving workerunit {$workerunit->_id} with activity {$workerunit->activity_id}.");
} catch (Exception $e) {
if ($activity) {
$activity->forceDelete();
}
//if($workerunit) $workerunit->forceDelete();
throw new Exception('Error saving activity for workerunit.');
}
}
});
}
public function getWorkerunitdata($action = null)
{
if (is_null($action)) {
$entities = \MongoDB\Entity::where('documentType', 'csvresult')->where('title', 'like', '%workerunit%')->get();
if (count($entities) > 0) {
return View::make('preprocess.csvresult.pages.workerunitdata', compact('entities'));
}
return Redirect::to('files/upload')->with('flashNotice', 'You have not uploaded any "csvresult" documents yet');
} elseif ($action == "preview") {
if ($URI = Input::get('URI')) {
if ($entity = $this->repository->find($URI)) {
if ($entity->documentType == "csvresult") {
return $document = $this->csvresultMapper->processWorkerunitData($entity, true);
}
}
}
} elseif ($action == "process") {
if ($URI = Input::get('URI')) {
if ($entity = $this->repository->find($URI)) {
if ($entity->documentType == "csvresult") {
return $document = $this->csvresultMapper->processWorkerunitData($entity);
}
}
}
}
}
public function getProcess()
{
if ($URI = Input::get('URI')) {
if ($entity = $this->repository->find($URI)) {
if ($entity->documentType != "fullvideo") {
continue;
}
$videoPreprocessing = $this->fullvideoStructurer->process($entity);
$status_processing = $this->fullvideoStructurer->store($entity, $videoPreprocessing);
if (isset($status_processing["keyframes"])) {
if (!isset($status_processing["keyframes"]['error'])) {
//update the parent
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('keyframes.count' => $status_processing["keyframes"]['success']["noEntitiesCreated"]));
}
echo "<pre>";
}
if (isset($status_processing["segments"])) {
if (!isset($status_processing["segments"]['error'])) {
//update the parent
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('segments.count' => $status_processing["segments"]['success']["noEntitiesCreated"]));
}
echo "<pre>";
}
if (isset($status_processing["keyframes"]['success']) && isset($status_processing["segments"]['success'])) {
return Redirect::back()->with('flashSuccess', 'Your video has been pre-processed in keyframes and video segments');
} else {
return Redirect::back()->with('flashError', 'An error occurred while the video was being pre-processed in keyframes and video segments');
}
}
} else {
return Redirect::back()->with('flashError', 'No valid URI given: ' . $URI);
}
}
public function getSearchFieldsAndValues($format, $domain)
{
$fields = array();
$fields['formats'] = array("text", "image", "video");
if (is_null($format)) {
$domains = Entity::distinct('domain')->get();
$usersInvolvedInEntities = array_flatten(Entity::distinct('user_id')->get()->toArray());
} else {
$domains = Entity::where('format', $format)->distinct('domain')->get();
$usersInvolvedInEntities = array_flatten(Entity::where('format', $format)->distinct('user_id')->get()->toArray());
if ($key = array_search($format, $fields['formats'])) {
$value = $fields['formats'][$key];
unset($fields['formats'][$key]);
array_unshift($fields['formats'], $value);
}
}
if (is_null($domain)) {
$documentTypes = Entity::where('format', $format)->distinct('documentType')->get();
} else {
$documentTypes = Entity::where('format', $format)->where('domain', $domain)->distinct('documentType')->get();
$usersInvolvedInEntities = array_flatten(Entity::where('format', $format)->where('domain', $domain)->distinct('user_id')->get()->toArray());
}
foreach ($usersInvolvedInEntities as $key => $user_id) {
$fields['userAgents'][$key] = User::find($user_id);
}
$fields['domains'] = array_flatten($domains->toArray());
$fields['documentTypes'] = array_flatten($documentTypes->toArray());
return $fields;
}
public function getActions()
{
// get all uploaded documents
// TODO: change to select by actual type
$entities = \MongoDB\Entity::where('activity_id', 'LIKE', '%fileuploader%')->get();
if (count($entities) > 0) {
return View::make('media.preprocess.relex.pages.actions', compact('entities'));
}
return Redirect::to('media/upload')->with('flashNotice', 'You have not uploaded any documents yet');
}
public static function boot()
{
parent::boot();
static::creating(function ($jobconf) {
// IFEXISTS CHECK IS NOT HERE.
try {
$c = $jobconf->content;
if (isset($c['reward'])) {
$c['reward'] = (double) $c['reward'];
}
if (isset($c['hitLifetimeInMinutes'])) {
$c['hitLifetimeInMinutes'] = intval($c['hitLifetimeInMinutes']);
}
if (isset($c['autoApprovalDelayInMinutes'])) {
$c['autoApprovalDelayInMinutes'] = intval($c['autoApprovalDelayInMinutes']);
}
if (isset($c['expirationInMinutes'])) {
$c['expirationInMinutes'] = intval($c['expirationInMinutes']);
}
if (isset($c['workerunitsPerUnit'])) {
$c['workerunitsPerUnit'] = intval($c['workerunitsPerUnit']);
}
if (isset($c['unitsPerTask'])) {
$c['unitsPerTask'] = intval($c['unitsPerTask']);
if ($c['unitsPerTask'] == 0) {
$c['unitsPerTask'] = 1;
}
}
$jobconf->content = $c;
} catch (Exception $e) {
if ($jobconf) {
$jobconf->forceDelete();
}
throw new Exception('Error saving JobConfiguration.');
}
if (empty($jobconf->activity_id)) {
try {
$activity = new Activity();
$activity->label = "JobConfiguration is saved.";
$activity->softwareAgent_id = 'jobcreator';
$activity->save();
$jobconf->activity_id = $activity->_id;
Log::debug("Saved JobConfiguration with activity {$jobconf->activity_id}.");
} catch (Exception $e) {
if ($activity) {
$activity->forceDelete();
}
if ($jobconf) {
$jobconf->forceDelete();
}
throw new Exception('Error saving activity for JobConfiguration.');
}
}
});
}
/**
* Return view for selecting a document for preprocessing.
*/
public function getIndex()
{
$entities = Entity::where('activity_id', 'LIKE', '%fileuploader%')->get();
$thisUser = \Auth::user();
foreach ($entities as $ent) {
$hasPermission = PermissionHandler::checkProject($thisUser, $ent['project'], Permissions::PROJECT_WRITE);
$ent['canWrite'] = $hasPermission;
}
if (count($entities) > 0) {
return View::make('media.preprocess.text.pages.actions', compact('entities'));
}
return Redirect::to('media/upload')->with('flashNotice', 'You have not uploaded any documents yet');
}
public function anyView()
{
$jobArray = explode(',', Input::get('jobs'));
$colors = array();
$jobsInfo = array();
for ($iter = 0; $iter < count($jobArray); ++$iter) {
$jobID = $jobArray[$iter];
$jobsInfo[$jobID] = \MongoDB\Entity::find($jobID);
$color = AnalyticsController::$colorList[$iter % count(AnalyticsController::$colorList)];
$jobsInfo[$jobID]['color'] = $color;
$colors[$jobID] = $color;
}
return View::make('analytics.jobview')->with('jobConfigurations', $jobsInfo)->with('jobIDs', $jobArray)->with('jobColors', $colors);
}
/**
*
* @param array $input
* @return result status structure containing
* 'status' 'ok' or 'error'
* 'message' 'A status message'
* 'batch' the batch or null if error
*/
public function store(array $input)
{
try {
$this->createBatchCreatorSoftwareAgent();
} catch (Exception $e) {
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "batchcreator";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
try {
$entity = new Entity();
$entity->title = $input['batch_title'];
$entity->format = $input['format'];
$entity->domain = $input['domain'];
$entity->documentType = 'batch';
$entity->softwareAgent_id = 'batchcreator';
$entity->parents = $input['units'];
$entity->content = $input['batch_description'];
$entity->hash = md5(serialize($entity->parents));
$entity->activity_id = $activity->_id;
$entity->save();
Queue::push('Queues\\UpdateUnits', $input['units']);
return ['status' => 'ok', 'message' => $input['batch_title'] . " batch was successfully created. (URI: {$entity->_id})", 'batch' => $entity];
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
}
public static function boot()
{
parent::boot();
static::saving(function ($job) {
\Log::debug('Clearing jobCache and mainSearchFilters.');
\MongoDB\Temp::whereIn('_id', ['mainSearchFilters', 'jobCache', $job->_id])->forceDelete();
});
static::creating(function ($job) {
try {
if (!SoftwareAgent::find('jobcreator')) {
$softwareAgent = new SoftwareAgent();
$softwareAgent->_id = 'jobcreator';
$softwareAgent->label = "Job creation";
}
if (!isset($job->projectedCost) and !isset($job->iamemptyjob)) {
$reward = $job->jobConfiguration->content['reward'];
$workerunitsPerUnit = intval($job->jobConfiguration->content['workerunitsPerUnit']);
$unitsPerTask = intval($job->jobConfiguration->content['unitsPerTask']);
$unitsCount = count($job->batch->wasDerivedFrom);
if (!$unitsPerTask) {
$projectedCost = 0;
} else {
$projectedCost = round($reward / $unitsPerTask * ($unitsCount * $workerunitsPerUnit), 2);
}
$job->expectedWorkerunitsCount = $unitsCount * $job->jobConfiguration->content['workerunitsPerUnit'];
$job->projectedCost = $projectedCost;
}
$job->unitsCount = count($job->batch->wasDerivedFrom);
$job->latestMetrics = 0;
$job->workerunitsCount = 0;
$job->completion = 0.0;
// 0.00-1.00
if (!isset($job->activity_id)) {
$activity = new Activity();
$activity->label = "Job is uploaded to crowdsourcing platform.";
$activity->softwareAgent_id = 'jobcreator';
// TODO: JOB softwareAgent_id = $platform. Does this need to be the same?
$activity->save();
$job->activity_id = $activity->_id;
}
} catch (Exception $e) {
// Something went wrong with creating the Entity
$job->forceDelete();
throw $e;
}
Log::debug("Saved entity {$job->_id} with activity {$job->activity_id}.");
});
}
/**
* Fetch the last existing ID for the given format / domain / docType combination.
*/
private function getLastDocumentInc($format, $domain, $docType)
{
$lastMongoURIUsed = Entity::where('format', $format)->where('domain', $domain)->where('documentType', $docType)->get(array("_id"));
if (count($lastMongoURIUsed) > 0) {
$lastMongoURIUsed = $lastMongoURIUsed->sortBy(function ($entity) {
return $entity->_id;
}, SORT_NATURAL)->toArray();
if (end($lastMongoURIUsed)) {
$lastMongoIDUsed = explode("/", end($lastMongoURIUsed)['_id']);
$inc = end($lastMongoIDUsed) + 1;
}
} else {
$inc = 0;
}
return $inc;
}
/**
* Run the database counter seeds.
*
* @return void
*/
public function run()
{
Eloquent::unguard();
$this->command->info('Seeding all activities...');
$seeds = $this->getSeeds(Activity::distinct('_id')->get());
foreach ($seeds as $name => $seed) {
$this->command->info('Seed: ' . $name . ' = ' . $seed);
$counter = new Counter();
$counter['_id'] = $name;
$counter['seq'] = $seed;
$counter->save();
}
$this->command->info('Seeding all entities...');
$seeds = $this->getSeeds(Entity::distinct('_id')->get());
foreach ($seeds as $name => $seed) {
$this->command->info('Seed: ' . $name . ' = ' . $seed);
$counter = new Counter();
$counter['_id'] = $name;
$counter['seq'] = $seed;
$counter->save();
}
}
public static function boot()
{
parent::boot();
static::saving(function ($questiontemplate) {
if (empty($questiontemplate->activity_id)) {
try {
$activity = new Activity();
$activity->label = "Questiontemplate is saved.";
$activity->softwareAgent_id = 'templatebuilder';
$activity->save();
$questiontemplate->activity_id = $activity->_id;
Log::debug("Saving QuestionTemplate {$questiontemplate->_id} with activity {$questiontemplate->activity_id}.");
} catch (Exception $e) {
if ($activity) {
$activity->forceDelete();
}
if ($questiontemplate) {
$questiontemplate->forceDelete();
}
throw new Exception('Error saving activity for QuestionTemplate.');
}
}
});
}
public function postFeatures()
{
$return = array('status' => 'ok');
$input = Input::get();
$domain = $input[1];
$type = $input[2];
// CREATE ACTIVITY FOR BATCH
$activity = new Activity();
$activity->label = "Images posted for processing.";
$activity->softwareAgent_id = 'imagegetter';
$activity->save();
// LOOP THROUGH IMAGES CREATE ENTITIES WITH ACTIVITY-ID FOR NEW IMAGES
$url_ids = "";
foreach ($input[0] as $img) {
\Log::debug(json_encode($img));
try {
$parse = parse_url($img['url']);
//$source = $parse['host'];
// Save images as parent
$image = new Entity();
$image->domain = $domain;
$image->format = "image";
$content = $image->content;
$content['url'] = $img['url'];
$content['title'] = $img['title'];
$content['height'] = $img['height'];
$content['width'] = $img['width'];
$content['description'] = $img['description'];
$content['author'] = $img['author'];
$image->content = $content;
$image->documentType = $type;
$image->source = "Rijksmuseum";
$image->tags = ['unit'];
$image->activity_id = $activity->_id;
$image->softwareAgent_id = "imagegetter";
// Take last part of URL as image title
$temp = explode('/', $img['url']);
//$image->title = end($temp);
// CHECK WHETHER URL EXISTS ALREADY
$hash = md5(serialize($image->content));
if ($existingid = Entity::where('hash', $hash)->pluck('_id')) {
$imageid = $existingid;
} else {
$image->hash = $hash;
$image->activity_id = $activity->_id;
\Log::debug(json_encode($image->toArray()));
$image->save();
$existingid = $image->_id;
}
$url_ids .= "{$img['url']} {$existingid} ";
} catch (Exception $e) {
//delete image
if (isset($image)) {
$image->forceDelete();
}
//delete activity
if (isset($activity)) {
$activity->forceDelete();
}
//Session::flash('flashError', $e->getMessage());
$return['error'] = $e->getMessage();
$return['status'] = 'bad';
\Log::debug($e->getMessage());
return $return;
}
// RUN PYTHON SCRIPT THAT CALLS APIs TO ADD FEATURES TO IMAGE
}
//return $url_ids;
try {
//$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getRijks.py " . $domain . " " . $type . " " . 4 . " " . "vogel";
$command = "/usr/bin/python2.7 " . base_path() . "/app/lib/getAPIS/getMany.py " . $domain . " " . $type . " " . Auth::user()->email . " " . $url_ids;
//$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getMany.py art painting http://lh3.ggpht.com/Q1GZTdmwa8iTLgdbu5uAgzovmLbb7lsYhG-QgVcoN8A-WJtIsNUo4-VyTMd9iKHLp-XNm812WyUaSgQdHdjQjDioJQI=s0 999";
//return $command;
\Log::debug("Running {$command}");
exec($command, $output, $error);
$return['oo'] = $output;
$return['ee'] = $error;
//$return['a'] = $a;
//throw $e; // for debugging.
//return $error;
} catch (Exception $e) {
//throw $e; // for debugging.
\Log::debug("ERROR: " . $e->getMessage());
$return['error'] = $e->getMessage();
$return['status'] = 'bad';
}
return $this->returnJson($return);
}
Route::get('/urlsurls', function () {
echo '-------- paintings -------' . PHP_EOL;
$results = \MongoDB\Entity::whereIn('documentType', ['painting'])->get(['content.url']);
$results2 = \MongoDB\Entity::whereIn('documentType', ['drawing'])->get(['content.url']);
foreach ($results as $result) {
echo $result['content']['url'] . ' ';
echo $result['_id'] . PHP_EOL;
}
echo PHP_EOL . PHP_EOL;
echo '-------- drawings-------' . PHP_EOL;
foreach ($results2 as $result) {
echo $result['content']['url'] . PHP_EOL;
echo $result['_id'] . PHP_EOL;
}
echo PHP_EOL . PHP_EOL . "[";
$results = \MongoDB\Entity::whereIn('documentType', ['painting', 'drawing'])->get();
foreach ($results as $result) {
echo $result . "," . PHP_EOL;
}
echo "]";
exit;
return Redirect::to('home');
});
// define routes
Route::get('home', 'PagesController@index');
Route::controller('api/v1', '\\Api\\v1\\apiController');
Route::controller('api/media', '\\Api\\media\\apiController');
Route::controller('api/search', '\\Api\\search\\apiController');
Route::controller('api/actions', '\\Api\\actions\\apiController');
Route::controller('api/analytics', '\\Api\\analytics\\apiController');
Route::get('login', 'UserController@login');
public function getTest($entity, $format, $domain, $docType, $incr)
{
$id = "{$entity}/{$format}/{$domain}/{$docType}/{$incr}";
$unit = MongoDB\Entity::id($id)->first();
echo "<h1>{$unit->_id}</h1>\n";
echo "-Sentence:{$unit->content['sentence']['formatted']}<br>\n";
echo "-Term1:{$unit->content['terms']['first']['formatted']}<br>\n";
echo "-Term2:{$unit->content['terms']['second']['formatted']}<br>\n";
echo "<hr>\r\n";
foreach (Workerunit::where('unit_id', $unit->_id)->where('softwareAgent_id', 'amt')->get() as $ann) {
$dic = $ann->createAnnotationVector();
echo "<table>";
foreach ($ann->content as $key => $value) {
echo "<tr><td><b>{$key}</b></td><td>{$value}</td></tr>\r\n";
}
echo "</table>";
echo "\r\nTERM1\r\n";
echo "<table>";
foreach ($dic['term1'] as $key => $value) {
echo "<tr><td><b>{$key}</b></td><td>{$value}</td></tr>\r\n";
}
echo "</table>";
echo "\r\nTERM2\r\n";
echo "<table>";
foreach ($dic['term2'] as $key => $value) {
echo "<tr><td><b>{$key}</b></td><td>{$value}</td></tr>\r\n";
}
echo "</table>";
echo "\r\n<hr>\r\n";
}
}
public function getProcesscrowdgames()
{
$gameJobs = Job::where('softwareAgent_id', 'DrDetectiveGamingPlatform')->get();
$activity = new Activity();
$activity->softwareAgent_id = 'DrDetectiveGamingPlatform';
$activity->save();
foreach ($gameJobs as $job) {
// $annotations = Entity::where('jobParents', $job['_id'])->get();
// Create one annotation vector for each image on the game
$images = Entity::where('jobParents', $job['_id'])->distinct('content.task_data')->get();
$annotationsSummary = [];
foreach ($images as $image) {
$imageName = $image[0];
// unpack data
$annotations = Entity::where('jobParents', $job['_id'])->where('content.task_data', $imageName)->get();
$annotations = $annotations->toArray();
// Create an array with all coordinates given for target image.
$coordinates = array_column(array_column(array_column($annotations, 'content'), 'response'), 'Coordinates');
$allCoordinates = [];
foreach ($coordinates as $coords) {
// Flatten to array of coords.
foreach ($coords as $c) {
$allCoordinates[] = $c;
}
}
$aggCoords = static::aggregateCoordinates($allCoordinates);
$annotationsSummary[] = ['image' => $imageName, 'aggregateCoordinates' => $aggCoords];
}
// process annotations for this job into an annotation vector...
$e = new Entity();
$e->jobParents = [$job['_id']];
$e->annotationVector = $annotationsSummary;
$e->documentType = 'annotationVector';
$e->activity_id = $activity->_id;
$e->softwareAgent_id = $job->softwareAgent_id;
$e->project = $job->project;
$e->user_id = $job->user_id;
$e->save();
}
return 'OK -- may need adjustments...';
}
public function getProcess()
{
if ($URI = Input::get('URI')) {
if ($entity = $this->repository->find($URI)) {
if ($entity->documentType != "metadatadescription") {
continue;
}
$metadataProcessing = $this->metadataAnnotationStructurer->process($entity);
$status_processing = $this->metadataAnnotationStructurer->store($entity, $metadataProcessing);
if (isset($status_processing["processAutomatedEventExtraction"])) {
if (!isset($status_processing["processAutomatedEventExtraction"]['error'])) {
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEvents' => true));
}
echo "<pre>";
}
if (isset($status_processing["thdapi"])) {
if (!isset($status_processing["thdapi"]['error'])) {
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
}
echo "<pre>";
}
if (isset($status_processing["textrazorapi"])) {
if (!isset($status_processing["textrazorapi"]['error'])) {
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
}
echo "<pre>";
}
if (isset($status_processing["semitagsapi"])) {
if (!isset($status_processing["semitagsapi"]['error'])) {
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
}
echo "<pre>";
}
if (isset($status_processing["nerdapi"])) {
if (!isset($status_processing["nerdapi"]['error'])) {
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
}
echo "<pre>";
}
if (isset($status_processing["lupediaapi"])) {
if (!isset($status_processing["lupediaapi"]['error'])) {
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
}
echo "<pre>";
}
if (isset($status_processing["dbpediaspotlightapi"])) {
if (!isset($status_processing["dbpediaspotlightapi"]['error'])) {
\MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
}
echo "<pre>";
}
if (isset($status_processing["processAutomatedEventExtraction"]['success'])) {
$this->createStatisticsForMetadatadescriptionCache($entity->_id);
return Redirect::back()->with('flashSuccess', 'Your video description has been pre-processed in named entities and putative events');
} else {
return Redirect::back()->with('flashError', 'An error occurred while the video description was being pre-processed in named entities and putative events');
}
}
} else {
return Redirect::back()->with('flashError', 'No valid URI given: ' . $URI);
}
}
public function store($format, $domain, $documentType, $parameters, $noOfVideos)
{
//fastcgi_finish_request();
$listOfVideoIdentifiers = array();
$this->listRecords($parameters, $noOfVideos, $listOfVideoIdentifiers);
// dd("done");
$status = array();
try {
$this->createOpenimagesVideoGetterSoftwareAgent();
} catch (Exception $e) {
$status['error']['OnlineData'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "openimagesgetter";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$status['error']['OnlineData'] = $e->getMessage();
$activity->forceDelete();
return $status;
}
$count["count"] = 0;
foreach ($listOfVideoIdentifiers as $video) {
$title = $video;
try {
$entity = new Entity();
$entity->_id = $entity->_id;
$entity->title = strtolower($title);
$entity->domain = $domain;
$entity->format = $format;
$entity->documentType = $documentType;
$entity->source = "openimages";
$videoMetadata = $this->getRecord($video, $parameters["metadataPrefix"]);
$entity->content = $videoMetadata["content"];
$parents = array();
$entity->parents = $parents;
$entity->tags = array("unit");
$entity->segments = $count;
$entity->keyframes = $count;
$entity->hash = md5(serialize([$entity->content]));
$entity->activity_id = $activity->_id;
$entity->save();
Queue::push('Queues\\UpdateUnits', [$entity->_id]);
$status['success'][$title] = $title . " was successfully uploaded. (URI: {$entity->_id})";
if (isset($status['success'])) {
$this->storeVideoDescription($entity);
// dd($this->storeVideoDescription($entity));
}
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
}
$status["recno"] = count($listOfVideoIdentifiers);
return $status;
}
<?php
// Get a list of titles and template types which are already in the database
// and put them to dropdown
$aTitles = array(null => '---');
$aTypes = array(null => '---');
$_format = unserialize(Session::get('batch'))->format;
$batchUnits = unserialize(Session::get('batch'))->parents;
$batchUnitContent = \MongoDB\Entity::where("_id", $batchUnits[0])->get()->first();
$unitAttributes = array();
$c = array_change_key_case(array_dot($batchUnitContent['content']), CASE_LOWER);
foreach ($c as $key => $val) {
$key = strtolower(str_replace('.', '_', $key));
$unitAttributes[$key] = $key;
}
// dd($unitAttributes);
$_aTitles = \MongoDB\Entity::where("documentType", "jobconf")->where("format", $_format)->distinct("content.title")->get();
$_aTitles = array_flatten($_aTitles->toArray());
foreach ($_aTitles as $key => $value) {
$pos = strpos($value, '[[');
if ($pos > 0) {
$t = trim(substr($value, 0, $pos));
if (!array_key_exists($t, $aTitles)) {
$aTitles[$t] = $t;
}
}
}
$_aTypes = \MongoDB\Template::where("format", $_format)->distinct('type')->get();
$_aTypes = array_flatten($_aTypes->toArray());
foreach ($_aTypes as $key => $value) {
if (!isset($aTypes[$value])) {
$aTypes[$value] = $value;
/**
*
* @param $data =
* array(unitids)
*/
public function fire($job, $data)
{
// TODO: some error handling
$jobIdsPerType = array();
foreach (Job::get() as $j) {
if (isset($jobIdsPerType[$j->type])) {
array_push($jobIdsPerType[$j->type], $j->_id);
} else {
$jobIdsPerType[$j->type] = [$j->_id];
}
}
foreach ($data as $id) {
set_time_limit(30);
$unit = \MongoDB\Entity::id($id)->first();
// dd($id);
$batch['count'] = count(\MongoDB\Entity::where('documentType', 'batch')->where('parents', 'all', array($unit->_id))->get()->toArray());
$workerunit = array('count' => 0, 'spam' => 0, 'nonSpam' => 0);
$workerlist = $workersspam = $workersnonspam = $joblist = array();
foreach (Workerunit::where('unit_id', $unit->_id)->get() as $a) {
$joblist[] = $a->job_id;
$workerlist[] = $a->crowdAgent_id;
if ($a->spam) {
$workerunit['spam']++;
$workersspam[] = $a->crowdAgent_id;
} else {
$workerunit['nonSpam']++;
$workersnonspam[] = $a->crowdAgent_id;
}
}
$workerunit['count'] = $workerunit['spam'] + $workerunit['nonSpam'];
$workers['count'] = count(array_unique($workerlist));
$workers['spam'] = count(array_unique($workersspam));
$workers['nonSpam'] = count(array_unique($workersnonspam));
$workers['potentialSpam'] = count(array_intersect($workersspam, $workersnonspam));
// Jobs
$jobs['count'] = count(array_unique($joblist));
foreach (array_keys($jobIdsPerType) as $type) {
$jobs['types'] = array();
$count = count(array_intersect(array_unique($joblist), $jobIdsPerType[$type]));
if ($count != 0) {
$jobs["types"][$type] = $count;
}
}
if (array_key_exists('types', $jobs)) {
$jobs['distinct'] = count($jobs['types']);
} else {
$jobs['distinct'] = 0;
}
$platformField = array();
$platformField['cf'] = count(\MongoDB\Entity::where('unit_id', $unit->_id)->where('softwareAgent_id', 'cf')->get()->toArray());
$platformField['amt'] = count(\MongoDB\Entity::where('unit_id', $unit->_id)->where('softwareAgent_id', 'amt')->get()->toArray());
// filtered
$filteredField = array();
$filteredField['job_ids'] = array_flatten(Job::where('metrics.filteredUnits.list', 'all', array($unit['_id']))->get(['_id'])->toArray());
$filteredField['count'] = count($filteredField['job_ids']);
$derivatives = \MongoDB\Entity::whereIn('parents', array($unit->_id))->lists('_id');
$children["count"] = count($derivatives);
$children["list"] = $derivatives;
$unit->cache = ["jobs" => $jobs, "workers" => $workers, "softwareAgent" => $platformField, "workerunits" => $workerunit, "filtered" => $filteredField, "batches" => $batch, "children" => $children];
$unit->update();
$avg_clarity = \MongoDB\Entity::where('metrics.units.withoutSpam.' . $unit->_id, 'exists', 'true')->avg('metrics.units.withoutSpam.' . $unit->id . '.max_relation_Cos.avg');
if (!isset($avg_clarity)) {
$avg_clarity = 0;
}
$unit->avg_clarity = $avg_clarity;
$unit->update();
\Log::debug("Updated unit {$unit->_id}.");
}
$job->delete();
// the Queue job...
}
public function getJobtypes()
{
return array_flatten(\MongoDB\Entity::where('documentType', 'job')->distinct('type')->get()->toArray());
}
public function storeVideoSegments($parentEntity, $videoSegmenting)
{
$tempEntityID = null;
$status = array();
try {
$this->createVideoSegmentingSoftwareAgent();
} catch (Exception $e) {
$status['error']['videosegmenting'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "videosegmenting";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error'][$title] = $e->getMessage();
return $status;
}
for ($i = 0; $i < sizeof($videoSegmenting); $i++) {
$videoSegmentName = explode("/", $videoSegmenting[$i]["storage_url"]);
$title = $videoSegmentName[sizeof($videoSegmentName) - 1];
try {
$entity = new Entity();
$entity->_id = $tempEntityID;
$entity->title = strtolower($title);
$entity->domain = $parentEntity->domain;
$entity->format = "video";
$entity->documentType = "videosegment";
$entity->parents = array($parentEntity->_id);
$entity->source = $parentEntity->source;
$entity->content = $videoSegmenting[$i];
//unset($relexStructuredSentenceKeyVal['properties']);
$entity->hash = md5(serialize($videoSegmenting[$i]));
$entity->activity_id = $activity->_id;
$entity->save();
$status['success'][$title] = $title . " was successfully processed into a video segment. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
$tempEntityID = $entity->_id;
}
$status['success']['noEntitiesCreated'] = sizeof($videoSegmenting);
//dd($status);
return $status;
}
public function updateStats2()
{
// take all the jobs for that worker
if ($crowdAgentJobs = Job::where('metrics.workers.withFilter.' . $this->_id, 'exists', true)->get(['_id'])) {
//if there is at least one job with that worker
if (count($crowdAgentJobs->toArray()) > 0) {
$domains = $formats = $types = $jobids = array();
$spam = $nonspam = $totalNoOfWorkerunits = 0;
foreach ($this->workerunits as $a) {
$totalNoOfWorkerunits++;
if ($a->spam) {
$spam++;
} else {
$nonspam++;
}
$domains[] = $a->domain;
$formats[] = $a->format;
$types[] = $a->type;
$jobids[] = $a->job_id;
$unitids[] = $a->unit_id;
}
// $this->WorkerunitStats = array('count'=>$total['count'], 'spam'=>$spam, 'nonspam'=>$nonspam);
$distinctWorkerunitTypes = array_unique($types);
// These actually are the Workerunit types
$distinctMediaFormats = array_unique($formats);
$distinctMediaDomains = array_unique($domains);
$workerParticipatedIn = count(array_unique($unitids));
$cache["workerunits"] = ["count" => $totalNoOfWorkerunits, "spam" => $spam, "nonspam" => $nonspam];
// take all distinct batches
$distinctBatchIds = \MongoDB\Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->distinct('batch_id')->get(['_id']);
$cache["mediaTypes"] = ["count" => count($distinctWorkerunitTypes), "types" => []];
foreach ($distinctBatchIds as $distinctBatchId) {
$batchParents = array_flatten(\MongoDB\Entity::where('_id', '=', $distinctBatchId[0])->lists('parents'));
//print_r($batchParents[0]);
$batchParentsType = \MongoDB\Entity::where('_id', '=', $batchParents[0])->distinct('documentType')->get(['documentType']);
//print_r(array_flatten($batchParentsType->toArray())[0]);
if (isset($cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]])) {
$cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]] = $cache["mediaTypes"][array_flatten($batchParentsType->toArray())[0]] + 1;
} else {
$cache["mediaTypes"]["types"] = [];
$cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]] = 1;
}
}
$cache["mediaTypes"]["distinct"] = sizeof(array_keys($cache["mediaTypes"]["types"]));
if (count($distinctWorkerunitTypes) > 0) {
$cache["jobTypes"] = ["distinct" => count($distinctWorkerunitTypes), "count" => count(array_unique($jobids)), "types" => []];
foreach ($distinctWorkerunitTypes as $distinctJobType) {
$distinctJobTypeCount = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->count();
$distinctJobTemplateTypes = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->distinct('template')->get()->toArray();
$countJobTemplateTypes = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->count();
//$cache["jobTypes"]["types"][$distinctJobType[0]] = [];
$cache["jobTypes"]["types"][$distinctJobType]['distinct'] = count($distinctJobTemplateTypes);
$cache["jobTypes"]["types"][$distinctJobType]['count'] = count($countJobTemplateTypes);
$cache["jobTypes"]["types"][$distinctJobType]["templates"] = [];
foreach ($distinctJobTemplateTypes as $distinctJobTemplateType) {
$distinctJobTemplateAndCount = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->where('template', $distinctJobTemplateType)->count();
$cache["jobTypes"]["types"][$distinctJobType]["templates"][$distinctJobTemplateType[0]] = $distinctJobTemplateAndCount;
}
}
}
if (count($distinctMediaFormats) > 0) {
$cache["mediaFormats"] = ["distinct" => count($distinctMediaFormats), "count" => $workerParticipatedIn, "formats" => []];
$cache["mediaDomains"] = ["distinct" => count($distinctMediaDomains), "count" => $workerParticipatedIn, "domains" => []];
foreach ($distinctMediaFormats as $distinctMediaFormat) {
$distinctMediaFormatAndCount = \MongoDB\Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->where('documentType', 'job')->where('format', $distinctMediaFormat)->count();
$cache["mediaFormats"]["formats"][$distinctMediaFormat] = $distinctMediaFormatAndCount;
}
foreach ($distinctMediaDomains as $distinctMediaDomain) {
$distinctMediaDomainAndCount = \MongoDB\Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->where('documentType', 'job')->where('domain', $distinctMediaDomain)->count();
$cache["mediaDomains"]["domains"][$distinctMediaDomain] = $distinctMediaDomainAndCount;
}
}
$jobsAsSpammer = \MongoDB\Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->whereIn('metrics.spammers.list', [$this->_id])->lists('platformJobId');
$cache["spammer"]["count"] = count($jobsAsSpammer);
$cache["spammer"]["jobs"] = array_flatten($jobsAsSpammer);
$this->cache = $cache;
$this->save();
} else {
$this->save();
}
}
}
public function postLoadt()
{
$jc_id = Session::get('jobconf_id_t');
$j_id = Session::get('job_id_t');
$jc = \MongoDB\Entity::where("_id", $jc_id)->first();
$j = \MongoDB\Entity::where("_id", $j_id)->first();
$jcco = $jc['content'];
$jcco['type'] = Input::get('templateType');
if ($jcco['type'] == null) {
return Redirect::back()->with('flashError', "form not filled in (type).");
}
// get a selected, newest jcbase
$maxi = \MongoDB\Template::where("type", $jcco['type'])->where("format", Session::get('format_t'))->max('version');
$jcbase = \MongoDB\Template::where("type", $jcco['type'])->where("format", Session::get('format_t'))->where('version', $maxi)->first();
if (!isset($jcbase)) {
Session::flash('flashError', "template not found");
return Redirect::to("jobs2/submit");
}
if (!isset($jcbase['cml'])) {
Session::flash('flashError', "No template details in this template");
return Redirect::to("jobs2/submit");
}
$jcco['cml'] = $jcbase['cml'];
if (isset($jcbase['css'])) {
$jcco['css'] = $jcbase['css'];
}
if (isset($jcbase['instructions'])) {
$jcco['instructions'] = $jcbase['instructions'];
}
if (isset($jcbase['js'])) {
$jcco['js'] = $jcbase['js'];
}
$jcco['template_id'] = $jcbase['_id'];
$pos = strpos($jcco['title'], '[[');
$title = substr($jcco['title'], 0, $pos);
$rest = substr($jcco['title'], strpos($jcco['title'], '(entity/'));
$jcco['title'] = $title . "[[" . $jcco['type'] . $rest;
$jc['content'] = $jcco;
$j['type'] = $jcco['type'];
$jc->save();
$j->save();
$platform = App::make('cf2');
//upadte
$platform->cfUpdate($j['platformJobId'], $jc);
$successmessage = "Job loaded.";
Session::flash('flashSuccess', $successmessage);
return Redirect::to("jobs");
}
/**
* refresh search index
*/
public function postRefreshindex()
{
$searchComponent = new MediaSearchComponent();
// amount of units to index per iteration
$batchsize = 500;
$from = Input::get('next');
$unitCount = Entity::whereIn('tags', ['unit'])->count();
// reset index on start
if ($from == 0) {
$searchComponent->clear();
}
// reduce last batch to remaining units
if ($from + $batchsize > $unitCount) {
$batchsize = $unitCount - $from;
}
// all units in this range
$units = Entity::distinct('_id')->where('tags', ['unit'])->skip($from)->take($batchsize)->get();
// get keys for each unit in this batch
$allKeys = [];
for ($i = $from; $i < $from + $batchsize; $i++) {
// get data of unit
$unit = Entity::where('_id', $units[$i][0])->first();
// map all properties into keys with formats
$keys = $this->getKeys($unit->attributesToArray());
// merge keys with set of keys and get the right format (e.g. if it occurs both at string and int we treat all of them as a string
foreach ($keys as $k => $v) {
if (!array_key_exists($k, $allKeys)) {
$allKeys[$k] = ['key' => $keys[$k]['key'], 'label' => $keys[$k]['label'], 'format' => $keys[$k]['format'], 'documents' => [$keys[$k]['document']]];
} else {
$allKeys[$k]['format'] = $searchComponent->prioritizeFormat([$allKeys[$k]['format'], $keys[$k]['format']]);
// add document type if its not in the list yet
if (!in_array($keys[$k]['document'], $allKeys[$k]['documents'])) {
array_push($allKeys[$k]['documents'], $keys[$k]['document']);
}
}
}
}
$searchComponent->store($allKeys);
return ['log' => $from . ' to ' . ($from + $batchsize) . ' of ' . $unitCount, 'next' => $from + $batchsize, 'last' => $unitCount];
}
public function getJobCountAttribute()
{
if ($this->documentType == "relex-structured-sentence") {
return $workerunits = count(array_flatten(Entity::where('unit_id', $this->_id)->distinct('job_id')->get()->toArray()));
}
}
public function getUnitsrelex()
{
$count = 0;
foreach (\Workerunit::where('type', 'RelEx')->get() as $ann) {
set_time_limit(30);
if (!isset($ann->content)) {
echo "{$ann->_id} no content\r\n";
echo "--------------------------------\r\n";
continue;
}
if (!empty($ann->unit_id)) {
echo "{$ann->_id} has unitid\r\n";
continue;
}
//dd($ann->question);
//$xml = simplexml_load_string($ann->question);
//$url = (string) $xml->ExternalURL;
$xml = simplexml_load_string($ann->question);
//$html = $ann->question;//
$html = (string) $xml->HTMLContent;
//dd($html);
$dom = HtmlDomParser::str_get_html($html);
$sentence = rtrim($dom->find('span[class=senval]', 0)->innertext, '.');
$term1 = $dom->find('span[style=color:#0000CD;]', 1)->innertext;
$term2 = $dom->find('span[style=color:#0000CD;]', 2)->innertext;
/* $sentence = "Poisson regression analysis which included data for multiple measurements of Tme/[TE] over the first year of life and adjusted for age-at-test and maternal smoking during [PREGNANCY] also demonstrated a greater decrease in Tme/Te in female infants who subsequently develop an LRI (P = 0.08";
$term1 = "[PREGNANCY]";
$term2 = "[TE]";*/
$unit = \MongoDB\Entity::where('content.terms.first.formatted', $term1)->where('content.terms.second.formatted', $term2)->where('content.sentence.formatted', $sentence)->first();
/* if(!$unit){
$hi = 0;
$units = \MongoDB\Entity::where('content.terms.first.formatted', $term1)
->where('content.terms.second.formatted', $term2)
->get();
foreach($units as $punit){
try{
$pct = similar_text($sentence, $punit->content['sentence']['formatted']);
} catch (ErrorException $e) {
echo "\r\n\r\n\r\n\r\n{$punit->_id}\r\n\r\n\r\n\r\n";
$pct = similar_text(strtolower($sentence), strtolower($punit->content['sentence']['text']));
}
if($pct>$hi) {
$hi=$pct;
$unit = $punit;
}
}
} */
if ($unit) {
echo "\r\n\\YY {$sentence}\r\n";
echo "== " . $unit->content['sentence']['formatted'];
echo "\r\n{$ann->_id}->{$unit->_id}\r\n";
$ann->unit_id = $unit->_id;
$ann->save();
} else {
echo "\r\nNO {$ann->_id}\r\n{$term1}--{$term2}--{$sentence}\r\n";
//echo $punit->content['sentence']['formatted'];
//echo "\r\n{$ann->unit_id}----------------\r\n";
echo "----------------------------------------";
continue;
}
}
}
public function OldcreateWorkerunitsAndCrowdAgents($mappedWorkerunitsWithUnits, $job_id, $taskType = "FactSpan")
{
$status = array();
$index = 0;
try {
$activity = new Activity();
$activity->softwareAgent_id = "cf";
$activity->save();
} catch (Exception $e) {
$activity->forceDelete();
$status['error'][$index]['activity'] = $e->getMessage();
}
foreach ($mappedWorkerunitsWithUnits as $mappedWorkerunitsWithUnit) {
$index++;
$crowdagent = CrowdAgent::where('platformAgentId', $mappedWorkerunitsWithUnit['_worker_id'])->where('softwareAgent_id', 'cf')->first();
if (!$crowdagent) {
try {
$crowdagent = new CrowdAgent();
$crowdagent->_id = "crowdagent/cf/" . $mappedWorkerunitsWithUnit['_worker_id'];
$crowdagent->softwareAgent_id = 'cf';
$crowdagent->platformAgentId = (int) $mappedWorkerunitsWithUnit['_worker_id'];
$crowdagent->country = $mappedWorkerunitsWithUnit['_country'];
$crowdagent->region = $mappedWorkerunitsWithUnit['_region'];
$crowdagent->city = $mappedWorkerunitsWithUnit['_city'];
$crowdagent->cfWorkerTrust = (double) $mappedWorkerunitsWithUnit['_trust'];
$crowdagent->save();
} catch (Exception $e) {
$status['error'][$index]['crowdagent'] = $e->getMessage();
// continue;
}
}
if (!Entity::where('softwareAgent_id', 'cf')->where('platformWorkerunitId', $mappedWorkerunitsWithUnit['_id'])->first()) {
$entity = new Entity();
$entity->format = "text";
$entity->domain = "medical";
$entity->documentType = "workerunit";
$entity->job_id = $job_id;
$entity->activity_id = $activity->_id;
$entity->crowdAgent_id = $crowdagent->_id;
$entity->softwareAgent_id = "cf";
$entity->unit_id = $mappedWorkerunitsWithUnit['unit']['_id'];
$entity->platformWorkerunitId = (int) $mappedWorkerunitsWithUnit['_id'];
$entity->cfChannel = $mappedWorkerunitsWithUnit['_channel'];
$entity->acceptTime = new MongoDate(strtotime($mappedWorkerunitsWithUnit['_started_at']));
$entity->submitTime = new MongoDate(strtotime($mappedWorkerunitsWithUnit['_created_at']));
$entity->cfTrust = (double) $mappedWorkerunitsWithUnit['_trust'];
if ($taskType == "FactSpan") {
$entity->content = ["confirmfirstfactor" => $mappedWorkerunitsWithUnit['confirmfirstfactor'], "confirmsecondfactor" => $mappedWorkerunitsWithUnit['confirmsecondfactor'], "firstfactor" => $mappedWorkerunitsWithUnit['firstfactor'], "secondfactor" => $mappedWorkerunitsWithUnit['secondfactor'], "saveselectionids1" => $mappedWorkerunitsWithUnit['saveselectionids1'], "saveselectionids2" => $mappedWorkerunitsWithUnit['saveselectionids2'], "confirmids1" => $mappedWorkerunitsWithUnit['confirmids1'], "confirmids2" => $mappedWorkerunitsWithUnit['confirmids2'], "sentencefirstfactor" => $mappedWorkerunitsWithUnit['sentencefirstfactor'], "sentencesecondfactor" => $mappedWorkerunitsWithUnit['sentencesecondfactor']];
} elseif ($taskType == "RelEx") {
$entity->content = ["step_1_select_the_valid_relations" => $mappedWorkerunitsWithUnit['step_1_select_the_valid_relations'], "step_2a_copy__paste_only_the_words_from_the_sentence_that_express_the_relation_you_selected_in_step1" => $mappedWorkerunitsWithUnit['step_2a_copy__paste_only_the_words_from_the_sentence_that_express_the_relation_you_selected_in_step1'], "step_2b_if_you_selected_none_in_step_1_explain_why" => $mappedWorkerunitsWithUnit['step_2b_if_you_selected_none_in_step_1_explain_why']];
} elseif ($taskType == "RelDir") {
$entity->content = ["direction" => $mappedWorkerunitsWithUnit['direction']];
}
try {
$entity->save();
} catch (Exception $e) {
$status['error'][$index]['entity'] = $e->getMessage();
}
}
}
return $status;
}
public function getLastDocumentInc()
{
$lastMongoURIUsed = Entity::where('format', 'text')->where('domain', 'medical')->where("documentType", 'relex-structured-sentence')->get(array("_id"));
if (count($lastMongoURIUsed) > 0) {
$lastMongoURIUsed = $lastMongoURIUsed->sortBy(function ($entity) {
return $entity->_id;
}, SORT_NATURAL)->toArray();
if (end($lastMongoURIUsed)) {
$lastMongoIDUsed = explode("/", end($lastMongoURIUsed)['_id']);
$inc = end($lastMongoIDUsed) + 1;
}
} else {
$inc = 0;
}
unset($lastMongoURIUsed);
return $inc;
}
