public function getWorkerunitdata($action = null)
 {
     if (is_null($action)) {
         $entities = \MongoDB\Entity::where('documentType', 'csvresult')->where('title', 'like', '%workerunit%')->get();
         if (count($entities) > 0) {
             return View::make('preprocess.csvresult.pages.workerunitdata', compact('entities'));
         }
         return Redirect::to('files/upload')->with('flashNotice', 'You have not uploaded any "csvresult" documents yet');
     } elseif ($action == "preview") {
         if ($URI = Input::get('URI')) {
             if ($entity = $this->repository->find($URI)) {
                 if ($entity->documentType == "csvresult") {
                     return $document = $this->csvresultMapper->processWorkerunitData($entity, true);
                 }
             }
         }
     } elseif ($action == "process") {
         if ($URI = Input::get('URI')) {
             if ($entity = $this->repository->find($URI)) {
                 if ($entity->documentType == "csvresult") {
                     return $document = $this->csvresultMapper->processWorkerunitData($entity);
                 }
             }
         }
     }
 }
 public function getProcess()
 {
     if ($URI = Input::get('URI')) {
         if ($entity = $this->repository->find($URI)) {
             if ($entity->documentType != "fullvideo") {
                 continue;
             }
             $videoPreprocessing = $this->fullvideoStructurer->process($entity);
             $status_processing = $this->fullvideoStructurer->store($entity, $videoPreprocessing);
             if (isset($status_processing["keyframes"])) {
                 if (!isset($status_processing["keyframes"]['error'])) {
                     //update the parent
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('keyframes.count' => $status_processing["keyframes"]['success']["noEntitiesCreated"]));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["segments"])) {
                 if (!isset($status_processing["segments"]['error'])) {
                     //update the parent
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('segments.count' => $status_processing["segments"]['success']["noEntitiesCreated"]));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["keyframes"]['success']) && isset($status_processing["segments"]['success'])) {
                 return Redirect::back()->with('flashSuccess', 'Your video has been pre-processed in keyframes and video segments');
             } else {
                 return Redirect::back()->with('flashError', 'An error occurred while the video was being pre-processed in keyframes and video segments');
             }
         }
     } else {
         return Redirect::back()->with('flashError', 'No valid URI given: ' . $URI);
     }
 }
Example #3
0
 public function getSearchFieldsAndValues($format, $domain)
 {
     $fields = array();
     $fields['formats'] = array("text", "image", "video");
     if (is_null($format)) {
         $domains = Entity::distinct('domain')->get();
         $usersInvolvedInEntities = array_flatten(Entity::distinct('user_id')->get()->toArray());
     } else {
         $domains = Entity::where('format', $format)->distinct('domain')->get();
         $usersInvolvedInEntities = array_flatten(Entity::where('format', $format)->distinct('user_id')->get()->toArray());
         if ($key = array_search($format, $fields['formats'])) {
             $value = $fields['formats'][$key];
             unset($fields['formats'][$key]);
             array_unshift($fields['formats'], $value);
         }
     }
     if (is_null($domain)) {
         $documentTypes = Entity::where('format', $format)->distinct('documentType')->get();
     } else {
         $documentTypes = Entity::where('format', $format)->where('domain', $domain)->distinct('documentType')->get();
         $usersInvolvedInEntities = array_flatten(Entity::where('format', $format)->where('domain', $domain)->distinct('user_id')->get()->toArray());
     }
     foreach ($usersInvolvedInEntities as $key => $user_id) {
         $fields['userAgents'][$key] = User::find($user_id);
     }
     $fields['domains'] = array_flatten($domains->toArray());
     $fields['documentTypes'] = array_flatten($documentTypes->toArray());
     return $fields;
 }
Example #4
0
 public function getActions()
 {
     // get all uploaded documents
     // TODO: change to select by actual type
     $entities = \MongoDB\Entity::where('activity_id', 'LIKE', '%fileuploader%')->get();
     if (count($entities) > 0) {
         return View::make('media.preprocess.relex.pages.actions', compact('entities'));
     }
     return Redirect::to('media/upload')->with('flashNotice', 'You have not uploaded any documents yet');
 }
Example #5
0
 /**
  * Return view for selecting a document for preprocessing.
  */
 public function getIndex()
 {
     $entities = Entity::where('activity_id', 'LIKE', '%fileuploader%')->get();
     $thisUser = \Auth::user();
     foreach ($entities as $ent) {
         $hasPermission = PermissionHandler::checkProject($thisUser, $ent['project'], Permissions::PROJECT_WRITE);
         $ent['canWrite'] = $hasPermission;
     }
     if (count($entities) > 0) {
         return View::make('media.preprocess.text.pages.actions', compact('entities'));
     }
     return Redirect::to('media/upload')->with('flashNotice', 'You have not uploaded any documents yet');
 }
 /**
  * Fetch the last existing ID for the given format / domain / docType combination.
  */
 private function getLastDocumentInc($format, $domain, $docType)
 {
     $lastMongoURIUsed = Entity::where('format', $format)->where('domain', $domain)->where('documentType', $docType)->get(array("_id"));
     if (count($lastMongoURIUsed) > 0) {
         $lastMongoURIUsed = $lastMongoURIUsed->sortBy(function ($entity) {
             return $entity->_id;
         }, SORT_NATURAL)->toArray();
         if (end($lastMongoURIUsed)) {
             $lastMongoIDUsed = explode("/", end($lastMongoURIUsed)['_id']);
             $inc = end($lastMongoIDUsed) + 1;
         }
     } else {
         $inc = 0;
     }
     return $inc;
 }
Example #7
0
 public function getVector()
 {
     if (($handle = fopen(storage_path() . '/output_AMT_FactSpan_sentences_raw.csv', 'r')) === false) {
         die('Error opening file');
     }
     $headers = fgetcsv($handle, 1024, ',');
     $count = 0;
     $complete = array();
     $return = array();
     $skip = false;
     while ($row = fgetcsv($handle, 1024, ',')) {
         set_time_limit(30);
         $skip = !$skip;
         if ($skip) {
             continue;
         }
         $count++;
         $c = array_combine($headers, $row);
         $sentence = rtrim($c['sentence'], '.');
         $term1 = $c['term1'];
         $term2 = $c['term2'];
         $found = false;
         foreach (MongoDB\Entity::where('documentType', 'relex-structured-sentence')->get() as $unit) {
             if ($unit['content']['sentence']['formatted'] == $sentence and $unit['content']['terms']['first']['formatted'] == $term1 and $unit['content']['terms']['second']['formatted'] == $term2) {
                 $found = true;
                 // THis can also be used to compare with CF.
                 //foreach (Job::type('FactSpan')->where('softwareAgent_id', 'amt')->get() as $job) {
                 $job = Job::id('entity/text/medical/job/0')->first();
                 if (in_array($unit->_id, array_keys($job->results))) {
                     $vector = $job->results[$unit->_id];
                     if (isset($vector['term1'])) {
                         //$temp = $this->computeSimilarity($vector['term1'], 1, $unit->_id, $job->softwareAgent_id);
                         $temp = $vector['term1'];
                         $temp['sentence'] = $sentence;
                         $temp['term1'] = $term1;
                         $temp['term2'] = $term2;
                         $result[] = $temp;
                     }
                     if (isset($vector['term2'])) {
                         //$temp = $this->computeSimilarity($vector['term2'], 2, $unit->_id, $job->softwareAgent_id);
                         $temp = $vector['term2'];
                         $temp['sentence'] = $sentence;
                         $temp['term1'] = $term1;
                         $temp['term2'] = $term2;
                         $result[] = $temp;
                     }
                 }
             }
             //}
         }
         if (!$found) {
             $result[] = array('', '', '', '', '', '', '', '', '', '', '', '', '', '');
             $result[] = array('', '', '', '', '', '', '', '', '', '', '', '', '', '');
         }
         //if($count==5) dd($result);
         /*	
         	if($count == 5){
         		dd($result);
         		$path =storage_path() . '/amt_new_output.csv';
         		$out = fopen($path, 'w');
         
         		fputcsv($out, array_keys($result[0]));
         		foreach ($result as $row)
         			fputcsv($out, $row);	
         		
         		// Close file
         		rewind($out);
         		fclose($out);
         		dd($path);
         	}*/
     }
     $path = storage_path() . '/amt_new_output.csv';
     $out = fopen($path, 'w');
     fputcsv($out, array_keys($result[0]));
     foreach ($result as $row) {
         fputcsv($out, $row);
     }
     // Close file
     rewind($out);
     fclose($out);
     dd($path);
 }
Example #8
0
 public function postFeatures()
 {
     $return = array('status' => 'ok');
     $input = Input::get();
     $domain = $input[1];
     $type = $input[2];
     // CREATE ACTIVITY FOR BATCH
     $activity = new Activity();
     $activity->label = "Images posted for processing.";
     $activity->softwareAgent_id = 'imagegetter';
     $activity->save();
     // LOOP THROUGH IMAGES CREATE ENTITIES WITH ACTIVITY-ID FOR NEW IMAGES
     $url_ids = "";
     foreach ($input[0] as $img) {
         \Log::debug(json_encode($img));
         try {
             $parse = parse_url($img['url']);
             //$source = $parse['host'];
             // Save images as parent
             $image = new Entity();
             $image->domain = $domain;
             $image->format = "image";
             $content = $image->content;
             $content['url'] = $img['url'];
             $content['title'] = $img['title'];
             $content['height'] = $img['height'];
             $content['width'] = $img['width'];
             $content['description'] = $img['description'];
             $content['author'] = $img['author'];
             $image->content = $content;
             $image->documentType = $type;
             $image->source = "Rijksmuseum";
             $image->tags = ['unit'];
             $image->activity_id = $activity->_id;
             $image->softwareAgent_id = "imagegetter";
             // Take last part of URL as image title
             $temp = explode('/', $img['url']);
             //$image->title = end($temp);
             // CHECK WHETHER URL EXISTS ALREADY
             $hash = md5(serialize($image->content));
             if ($existingid = Entity::where('hash', $hash)->pluck('_id')) {
                 $imageid = $existingid;
             } else {
                 $image->hash = $hash;
                 $image->activity_id = $activity->_id;
                 \Log::debug(json_encode($image->toArray()));
                 $image->save();
                 $existingid = $image->_id;
             }
             $url_ids .= "{$img['url']} {$existingid} ";
         } catch (Exception $e) {
             //delete image
             if (isset($image)) {
                 $image->forceDelete();
             }
             //delete activity
             if (isset($activity)) {
                 $activity->forceDelete();
             }
             //Session::flash('flashError', $e->getMessage());
             $return['error'] = $e->getMessage();
             $return['status'] = 'bad';
             \Log::debug($e->getMessage());
             return $return;
         }
         // RUN PYTHON SCRIPT THAT CALLS APIs TO ADD FEATURES TO IMAGE
     }
     //return $url_ids;
     try {
         //$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getRijks.py " . $domain . " " . $type . " " . 4 . " " . "vogel";
         $command = "/usr/bin/python2.7 " . base_path() . "/app/lib/getAPIS/getMany.py " . $domain . " " . $type . " " . Auth::user()->email . " " . $url_ids;
         //$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getMany.py art painting http://lh3.ggpht.com/Q1GZTdmwa8iTLgdbu5uAgzovmLbb7lsYhG-QgVcoN8A-WJtIsNUo4-VyTMd9iKHLp-XNm812WyUaSgQdHdjQjDioJQI=s0 999";
         //return $command;
         \Log::debug("Running {$command}");
         exec($command, $output, $error);
         $return['oo'] = $output;
         $return['ee'] = $error;
         //$return['a'] = $a;
         //throw $e; // for debugging.
         //return $error;
     } catch (Exception $e) {
         //throw $e; // for debugging.
         \Log::debug("ERROR: " . $e->getMessage());
         $return['error'] = $e->getMessage();
         $return['status'] = 'bad';
     }
     return $this->returnJson($return);
 }
 public function getProcess()
 {
     if ($URI = Input::get('URI')) {
         if ($entity = $this->repository->find($URI)) {
             if ($entity->documentType != "metadatadescription") {
                 continue;
             }
             $metadataProcessing = $this->metadataAnnotationStructurer->process($entity);
             $status_processing = $this->metadataAnnotationStructurer->store($entity, $metadataProcessing);
             if (isset($status_processing["processAutomatedEventExtraction"])) {
                 if (!isset($status_processing["processAutomatedEventExtraction"]['error'])) {
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEvents' => true));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["thdapi"])) {
                 if (!isset($status_processing["thdapi"]['error'])) {
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["textrazorapi"])) {
                 if (!isset($status_processing["textrazorapi"]['error'])) {
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["semitagsapi"])) {
                 if (!isset($status_processing["semitagsapi"]['error'])) {
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["nerdapi"])) {
                 if (!isset($status_processing["nerdapi"]['error'])) {
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["lupediaapi"])) {
                 if (!isset($status_processing["lupediaapi"]['error'])) {
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["dbpediaspotlightapi"])) {
                 if (!isset($status_processing["dbpediaspotlightapi"]['error'])) {
                     \MongoDB\Entity::where('_id', '=', $entity->_id)->update(array('preprocessed.automatedEntities' => true));
                 }
                 echo "<pre>";
             }
             if (isset($status_processing["processAutomatedEventExtraction"]['success'])) {
                 $this->createStatisticsForMetadatadescriptionCache($entity->_id);
                 return Redirect::back()->with('flashSuccess', 'Your video description has been pre-processed in named entities and putative events');
             } else {
                 return Redirect::back()->with('flashError', 'An error occurred while the video description was being pre-processed in named entities and putative events');
             }
         }
     } else {
         return Redirect::back()->with('flashError', 'No valid URI given: ' . $URI);
     }
 }
Example #10
0
 public function getProcesscrowdgames()
 {
     $gameJobs = Job::where('softwareAgent_id', 'DrDetectiveGamingPlatform')->get();
     $activity = new Activity();
     $activity->softwareAgent_id = 'DrDetectiveGamingPlatform';
     $activity->save();
     foreach ($gameJobs as $job) {
         // $annotations = Entity::where('jobParents', $job['_id'])->get();
         // Create one annotation vector for each image on the game
         $images = Entity::where('jobParents', $job['_id'])->distinct('content.task_data')->get();
         $annotationsSummary = [];
         foreach ($images as $image) {
             $imageName = $image[0];
             // unpack data
             $annotations = Entity::where('jobParents', $job['_id'])->where('content.task_data', $imageName)->get();
             $annotations = $annotations->toArray();
             // Create an array with all coordinates given for target image.
             $coordinates = array_column(array_column(array_column($annotations, 'content'), 'response'), 'Coordinates');
             $allCoordinates = [];
             foreach ($coordinates as $coords) {
                 // Flatten to array of coords.
                 foreach ($coords as $c) {
                     $allCoordinates[] = $c;
                 }
             }
             $aggCoords = static::aggregateCoordinates($allCoordinates);
             $annotationsSummary[] = ['image' => $imageName, 'aggregateCoordinates' => $aggCoords];
         }
         // process annotations for this job into an annotation vector...
         $e = new Entity();
         $e->jobParents = [$job['_id']];
         $e->annotationVector = $annotationsSummary;
         $e->documentType = 'annotationVector';
         $e->activity_id = $activity->_id;
         $e->softwareAgent_id = $job->softwareAgent_id;
         $e->project = $job->project;
         $e->user_id = $job->user_id;
         $e->save();
     }
     return 'OK -- may need adjustments...';
 }
Example #11
0
 public function getLastDocumentInc()
 {
     $lastMongoURIUsed = Entity::where('format', 'text')->where('domain', 'medical')->where("documentType", 'relex-structured-sentence')->get(array("_id"));
     if (count($lastMongoURIUsed) > 0) {
         $lastMongoURIUsed = $lastMongoURIUsed->sortBy(function ($entity) {
             return $entity->_id;
         }, SORT_NATURAL)->toArray();
         if (end($lastMongoURIUsed)) {
             $lastMongoIDUsed = explode("/", end($lastMongoURIUsed)['_id']);
             $inc = end($lastMongoIDUsed) + 1;
         }
     } else {
         $inc = 0;
     }
     unset($lastMongoURIUsed);
     return $inc;
 }
Example #12
0
 public function listRecords($parameters, $noEntries, &$listOfRecords)
 {
     $curlRequest = new SVRequest();
     $url = $this->url . "verb=ListRecords";
     if (isset($parameters)) {
         if (!array_key_exists("metadataPrefix", $parameters)) {
             if (!array_key_exists("resumptionToken", $parameters)) {
                 throw new Exception("Request must contain -metadataPrefix- parameter!");
             } else {
                 foreach ($parameters as $param => $value) {
                     $url .= "&" . $param . "=" . $value;
                 }
             }
         } else {
             foreach ($parameters as $param => $value) {
                 $url .= "&" . $param . "=" . $value;
             }
         }
     } else {
         throw new Exception('Request parameters missing!');
     }
     $entities = \MongoDB\Entity::where('documentType', 'fullvideo')->lists("title");
     while ($noEntries > 0) {
         //throw new Exception($noEntries);
         $result = $curlRequest->curlRequest($url, "POST", null);
         $xml = simplexml_load_string($result["result"]);
         if ($xml === false) {
             die('Error parsing XML');
         } else {
             $xmlNode = $xml->ListRecords;
             if (isset($xmlNode)) {
                 foreach ($xmlNode->record as $rNode) {
                     if (strpos((string) $rNode->metadata->children('oai_oi', 1)->oi->children('oi', 1)->type, "Moving Image") !== false) {
                         if ($noEntries > 0) {
                             if (!in_array((string) $rNode->header->identifier, $entities)) {
                                 array_push($listOfRecords, (string) $rNode->header->identifier);
                                 array_push($entities, (string) $rNode->header->identifier);
                                 //	dd($entities);
                                 $noEntries--;
                             } else {
                                 continue;
                             }
                         } else {
                             break;
                         }
                     }
                 }
             }
             if (isset($xml->ListRecords->resumptionToken)) {
                 if ($noEntries > 0) {
                     if (!array_key_exists("resumptionToken", $parameters)) {
                         $parameters["resumptionToken"] = (string) $xml->ListRecords->resumptionToken;
                         unset($parameters["metadataPrefix"]);
                         $this->listRecords($parameters, $noEntries, $listOfRecords);
                     } else {
                         $replacement = array("resumptionToken" => (string) $xml->ListRecords->resumptionToken);
                         $parameters = array_replace($parameters, $replacement);
                         $this->listRecords($parameters, $noEntries, $listOfRecords);
                     }
                 }
             }
         }
     }
     //	dd($listOfRecords);
     //	return $listOfRecords;
 }
Example #13
0
 public function getJobCountAttribute()
 {
     if ($this->documentType == "relex-structured-sentence") {
         return $workerunits = count(array_flatten(Entity::where('unit_id', $this->_id)->distinct('job_id')->get()->toArray()));
     }
 }
Example #14
0
 public static function createMainSearchFiltersCache()
 {
     // $mainSearchFilters['media']['formats'] = $this->getDistinctFieldAndCount('format', ['unit']);
     // $mainSearchFilters['media']['domains'] = $this->getDistinctFieldAndCount('domain', ['unit']);
     $mainSearchFilters['media']['categories'] = static::getCategories();
     $mainSearchFilters['media']['all'] = ["count" => \MongoDB\Entity::whereIn('tags', ['unit'])->count(), "label" => "All Media"];
     //unset($mainSearchFilters['media']['documentTypes']['twrex']);
     $mainSearchFilters['job']['count'] = Entity::where('documentType', 'job')->count();
     $mainSearchFilters['workers']['count'] = \MongoDB\CrowdAgent::all()->count();
     ksort($mainSearchFilters['media']['categories']);
     $entity = new \MongoDB\Temp();
     $entity->_id = "mainSearchFilters";
     $entity->filters = $mainSearchFilters;
     $entity->save();
     return $entity->toArray();
 }
Example #15
0
 public function postLoadt()
 {
     $jc_id = Session::get('jobconf_id_t');
     $j_id = Session::get('job_id_t');
     $jc = \MongoDB\Entity::where("_id", $jc_id)->first();
     $j = \MongoDB\Entity::where("_id", $j_id)->first();
     $jcco = $jc['content'];
     $jcco['type'] = Input::get('templateType');
     if ($jcco['type'] == null) {
         return Redirect::back()->with('flashError', "form not filled in (type).");
     }
     // get a selected, newest jcbase
     $maxi = \MongoDB\Template::where("type", $jcco['type'])->where("format", Session::get('format_t'))->max('version');
     $jcbase = \MongoDB\Template::where("type", $jcco['type'])->where("format", Session::get('format_t'))->where('version', $maxi)->first();
     if (!isset($jcbase)) {
         Session::flash('flashError', "template not found");
         return Redirect::to("jobs2/submit");
     }
     if (!isset($jcbase['cml'])) {
         Session::flash('flashError', "No template details in this template");
         return Redirect::to("jobs2/submit");
     }
     $jcco['cml'] = $jcbase['cml'];
     if (isset($jcbase['css'])) {
         $jcco['css'] = $jcbase['css'];
     }
     if (isset($jcbase['instructions'])) {
         $jcco['instructions'] = $jcbase['instructions'];
     }
     if (isset($jcbase['js'])) {
         $jcco['js'] = $jcbase['js'];
     }
     $jcco['template_id'] = $jcbase['_id'];
     $pos = strpos($jcco['title'], '[[');
     $title = substr($jcco['title'], 0, $pos);
     $rest = substr($jcco['title'], strpos($jcco['title'], '(entity/'));
     $jcco['title'] = $title . "[[" . $jcco['type'] . $rest;
     $jc['content'] = $jcco;
     $j['type'] = $jcco['type'];
     $jc->save();
     $j->save();
     $platform = App::make('cf2');
     //upadte
     $platform->cfUpdate($j['platformJobId'], $jc);
     $successmessage = "Job loaded.";
     Session::flash('flashSuccess', $successmessage);
     return Redirect::to("jobs");
 }
Example #16
0
 public static function createJobCache()
 {
     \Session::flash('rawArray', 1);
     $db = \DB::getMongoDB();
     $db = $db->temp;
     $result = \MongoDB\Entity::where('documentType', 'job')->with('hasConfiguration')->get()->toArray();
     if (count($result) > 0) {
         try {
             \MongoDB\Temp::where('documentType', 'job')->forceDelete();
             $db->batchInsert($result, array('continueOnError' => true));
         } catch (Exception $e) {
             // ContinueOnError will still throw an exception on duplication, even though it continues, so we just move on.
         }
     }
     \Session::forget('rawArray');
 }
Example #17
0
 public function anyTest()
 {
     try {
         if (!($data = Input::get('data'))) {
             if (!($data = \Request::getContent())) {
                 return false;
             }
         }
         if (empty($data)) {
             return false;
         }
         $data = json_decode($data, true);
         $data['softwareAgent_id'] = strtolower($data['softwareAgent_id']);
         try {
             $this->createPostSoftwareAgent($data);
         } catch (Exception $e) {
             return serialize([$e->getMessage()]);
         }
         try {
             $activity = new Activity();
             $activity->softwareAgent_id = $data['softwareAgent_id'];
             $activity->save();
         } catch (Exception $e) {
             // Something went wrong with creating the Activity
             $activity->forceDelete();
             return serialize([$e->getMessage()]);
         }
         $entity = new Entity();
         $entity->format = 'image';
         $entity->domain = $data['domain'];
         $entity->tags = $data['tags'];
         $entity->documentType = $data['documentType'];
         $entity->softwareAgent_configuration = $data['softwareAgent_configuration'];
         if (isset($data['parents'])) {
             $entity->parents = $data['parents'];
         }
         $entity->content = $data['content'];
         if (isset($data['threshold'])) {
             $entity->threshold = $data['threshold'];
         }
         if (isset($data['relevantFeatures'])) {
             $entity->recognizedFeatures = $data['relevantFeatures'];
         }
         if (isset($data['hash'])) {
             $entity->hash = $data['hash'];
         } else {
             $entity->hash = md5(serialize($data['content']));
         }
         $entity->activity_id = $activity->_id;
         if (Entity::where('hash', $entity->hash)->first()) {
             //dd('asdasd');
         } else {
             $entity->save();
         }
         return Response::json($entity);
     } catch (Exception $e) {
         dd($e->getMessage());
     }
 }
Example #18
0
 public function updateStats2()
 {
     // take all the jobs for that worker
     if ($crowdAgentJobs = Job::where('metrics.workers.withFilter.' . $this->_id, 'exists', true)->get(['_id'])) {
         //if there is at least one job with that worker
         if (count($crowdAgentJobs->toArray()) > 0) {
             $domains = $formats = $types = $jobids = array();
             $spam = $nonspam = $totalNoOfWorkerunits = 0;
             foreach ($this->workerunits as $a) {
                 $totalNoOfWorkerunits++;
                 if ($a->spam) {
                     $spam++;
                 } else {
                     $nonspam++;
                 }
                 $domains[] = $a->domain;
                 $formats[] = $a->format;
                 $types[] = $a->type;
                 $jobids[] = $a->job_id;
                 $unitids[] = $a->unit_id;
             }
             // $this->WorkerunitStats = array('count'=>$total['count'], 'spam'=>$spam, 'nonspam'=>$nonspam);
             $distinctWorkerunitTypes = array_unique($types);
             // These actually are the Workerunit types
             $distinctMediaFormats = array_unique($formats);
             $distinctMediaDomains = array_unique($domains);
             $workerParticipatedIn = count(array_unique($unitids));
             $cache["workerunits"] = ["count" => $totalNoOfWorkerunits, "spam" => $spam, "nonspam" => $nonspam];
             // take all distinct batches
             $distinctBatchIds = \MongoDB\Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->distinct('batch_id')->get(['_id']);
             $cache["mediaTypes"] = ["count" => count($distinctWorkerunitTypes), "types" => []];
             foreach ($distinctBatchIds as $distinctBatchId) {
                 $batchParents = array_flatten(\MongoDB\Entity::where('_id', '=', $distinctBatchId[0])->lists('parents'));
                 //print_r($batchParents[0]);
                 $batchParentsType = \MongoDB\Entity::where('_id', '=', $batchParents[0])->distinct('documentType')->get(['documentType']);
                 //print_r(array_flatten($batchParentsType->toArray())[0]);
                 if (isset($cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]])) {
                     $cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]] = $cache["mediaTypes"][array_flatten($batchParentsType->toArray())[0]] + 1;
                 } else {
                     $cache["mediaTypes"]["types"] = [];
                     $cache["mediaTypes"]["types"][array_flatten($batchParentsType->toArray())[0]] = 1;
                 }
             }
             $cache["mediaTypes"]["distinct"] = sizeof(array_keys($cache["mediaTypes"]["types"]));
             if (count($distinctWorkerunitTypes) > 0) {
                 $cache["jobTypes"] = ["distinct" => count($distinctWorkerunitTypes), "count" => count(array_unique($jobids)), "types" => []];
                 foreach ($distinctWorkerunitTypes as $distinctJobType) {
                     $distinctJobTypeCount = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->count();
                     $distinctJobTemplateTypes = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->distinct('template')->get()->toArray();
                     $countJobTemplateTypes = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->type($distinctJobType)->count();
                     //$cache["jobTypes"]["types"][$distinctJobType[0]] = [];
                     $cache["jobTypes"]["types"][$distinctJobType]['distinct'] = count($distinctJobTemplateTypes);
                     $cache["jobTypes"]["types"][$distinctJobType]['count'] = count($countJobTemplateTypes);
                     $cache["jobTypes"]["types"][$distinctJobType]["templates"] = [];
                     foreach ($distinctJobTemplateTypes as $distinctJobTemplateType) {
                         $distinctJobTemplateAndCount = Job::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->where('template', $distinctJobTemplateType)->count();
                         $cache["jobTypes"]["types"][$distinctJobType]["templates"][$distinctJobTemplateType[0]] = $distinctJobTemplateAndCount;
                     }
                 }
             }
             if (count($distinctMediaFormats) > 0) {
                 $cache["mediaFormats"] = ["distinct" => count($distinctMediaFormats), "count" => $workerParticipatedIn, "formats" => []];
                 $cache["mediaDomains"] = ["distinct" => count($distinctMediaDomains), "count" => $workerParticipatedIn, "domains" => []];
                 foreach ($distinctMediaFormats as $distinctMediaFormat) {
                     $distinctMediaFormatAndCount = \MongoDB\Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->where('documentType', 'job')->where('format', $distinctMediaFormat)->count();
                     $cache["mediaFormats"]["formats"][$distinctMediaFormat] = $distinctMediaFormatAndCount;
                 }
                 foreach ($distinctMediaDomains as $distinctMediaDomain) {
                     $distinctMediaDomainAndCount = \MongoDB\Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->where('documentType', 'job')->where('domain', $distinctMediaDomain)->count();
                     $cache["mediaDomains"]["domains"][$distinctMediaDomain] = $distinctMediaDomainAndCount;
                 }
             }
             $jobsAsSpammer = \MongoDB\Entity::whereIn('_id', array_flatten($crowdAgentJobs->toArray()))->whereIn('metrics.spammers.list', [$this->_id])->lists('platformJobId');
             $cache["spammer"]["count"] = count($jobsAsSpammer);
             $cache["spammer"]["jobs"] = array_flatten($jobsAsSpammer);
             $this->cache = $cache;
             $this->save();
         } else {
             $this->save();
         }
     }
 }
Example #19
0
									<?php 
// Get a list of titles and template types which are already in the database
// and put them to dropdown
$aTitles = array(null => '---');
$aTypes = array(null => '---');
$_format = unserialize(Session::get('batch'))->format;
$batchUnits = unserialize(Session::get('batch'))->parents;
$batchUnitContent = \MongoDB\Entity::where("_id", $batchUnits[0])->get()->first();
$unitAttributes = array();
$c = array_change_key_case(array_dot($batchUnitContent['content']), CASE_LOWER);
foreach ($c as $key => $val) {
    $key = strtolower(str_replace('.', '_', $key));
    $unitAttributes[$key] = $key;
}
//	dd($unitAttributes);
$_aTitles = \MongoDB\Entity::where("documentType", "jobconf")->where("format", $_format)->distinct("content.title")->get();
$_aTitles = array_flatten($_aTitles->toArray());
foreach ($_aTitles as $key => $value) {
    $pos = strpos($value, '[[');
    if ($pos > 0) {
        $t = trim(substr($value, 0, $pos));
        if (!array_key_exists($t, $aTitles)) {
            $aTitles[$t] = $t;
        }
    }
}
$_aTypes = \MongoDB\Template::where("format", $_format)->distinct('type')->get();
$_aTypes = array_flatten($_aTypes->toArray());
foreach ($_aTypes as $key => $value) {
    if (!isset($aTypes[$value])) {
        $aTypes[$value] = $value;
Example #20
0
 public function getJobtypes()
 {
     return array_flatten(\MongoDB\Entity::where('documentType', 'job')->distinct('type')->get()->toArray());
 }
Example #21
0
 /**
  * refresh search index
  */
 public function postRefreshindex()
 {
     $searchComponent = new MediaSearchComponent();
     // amount of units to index per iteration
     $batchsize = 500;
     $from = Input::get('next');
     $unitCount = Entity::whereIn('tags', ['unit'])->count();
     // reset index on start
     if ($from == 0) {
         $searchComponent->clear();
     }
     // reduce last batch to remaining units
     if ($from + $batchsize > $unitCount) {
         $batchsize = $unitCount - $from;
     }
     // all units in this range
     $units = Entity::distinct('_id')->where('tags', ['unit'])->skip($from)->take($batchsize)->get();
     // get keys for each unit in this batch
     $allKeys = [];
     for ($i = $from; $i < $from + $batchsize; $i++) {
         // get data of unit
         $unit = Entity::where('_id', $units[$i][0])->first();
         // map all properties into keys with formats
         $keys = $this->getKeys($unit->attributesToArray());
         // merge keys with set of keys and get the right format (e.g. if it occurs both at string and int we treat all of them as a string
         foreach ($keys as $k => $v) {
             if (!array_key_exists($k, $allKeys)) {
                 $allKeys[$k] = ['key' => $keys[$k]['key'], 'label' => $keys[$k]['label'], 'format' => $keys[$k]['format'], 'documents' => [$keys[$k]['document']]];
             } else {
                 $allKeys[$k]['format'] = $searchComponent->prioritizeFormat([$allKeys[$k]['format'], $keys[$k]['format']]);
                 // add document type if its not in the list yet
                 if (!in_array($keys[$k]['document'], $allKeys[$k]['documents'])) {
                     array_push($allKeys[$k]['documents'], $keys[$k]['document']);
                 }
             }
         }
     }
     $searchComponent->store($allKeys);
     return ['log' => $from . ' to ' . ($from + $batchsize) . ' of ' . $unitCount, 'next' => $from + $batchsize, 'last' => $unitCount];
 }
Example #22
0
 /**
  *
  * @param $data =
  *        	array(unitids)
  */
 public function fire($job, $data)
 {
     // TODO: some error handling
     $jobIdsPerType = array();
     foreach (Job::get() as $j) {
         if (isset($jobIdsPerType[$j->type])) {
             array_push($jobIdsPerType[$j->type], $j->_id);
         } else {
             $jobIdsPerType[$j->type] = [$j->_id];
         }
     }
     foreach ($data as $id) {
         set_time_limit(30);
         $unit = \MongoDB\Entity::id($id)->first();
         // dd($id);
         $batch['count'] = count(\MongoDB\Entity::where('documentType', 'batch')->where('parents', 'all', array($unit->_id))->get()->toArray());
         $workerunit = array('count' => 0, 'spam' => 0, 'nonSpam' => 0);
         $workerlist = $workersspam = $workersnonspam = $joblist = array();
         foreach (Workerunit::where('unit_id', $unit->_id)->get() as $a) {
             $joblist[] = $a->job_id;
             $workerlist[] = $a->crowdAgent_id;
             if ($a->spam) {
                 $workerunit['spam']++;
                 $workersspam[] = $a->crowdAgent_id;
             } else {
                 $workerunit['nonSpam']++;
                 $workersnonspam[] = $a->crowdAgent_id;
             }
         }
         $workerunit['count'] = $workerunit['spam'] + $workerunit['nonSpam'];
         $workers['count'] = count(array_unique($workerlist));
         $workers['spam'] = count(array_unique($workersspam));
         $workers['nonSpam'] = count(array_unique($workersnonspam));
         $workers['potentialSpam'] = count(array_intersect($workersspam, $workersnonspam));
         // Jobs
         $jobs['count'] = count(array_unique($joblist));
         foreach (array_keys($jobIdsPerType) as $type) {
             $jobs['types'] = array();
             $count = count(array_intersect(array_unique($joblist), $jobIdsPerType[$type]));
             if ($count != 0) {
                 $jobs["types"][$type] = $count;
             }
         }
         if (array_key_exists('types', $jobs)) {
             $jobs['distinct'] = count($jobs['types']);
         } else {
             $jobs['distinct'] = 0;
         }
         $platformField = array();
         $platformField['cf'] = count(\MongoDB\Entity::where('unit_id', $unit->_id)->where('softwareAgent_id', 'cf')->get()->toArray());
         $platformField['amt'] = count(\MongoDB\Entity::where('unit_id', $unit->_id)->where('softwareAgent_id', 'amt')->get()->toArray());
         // filtered
         $filteredField = array();
         $filteredField['job_ids'] = array_flatten(Job::where('metrics.filteredUnits.list', 'all', array($unit['_id']))->get(['_id'])->toArray());
         $filteredField['count'] = count($filteredField['job_ids']);
         $derivatives = \MongoDB\Entity::whereIn('parents', array($unit->_id))->lists('_id');
         $children["count"] = count($derivatives);
         $children["list"] = $derivatives;
         $unit->cache = ["jobs" => $jobs, "workers" => $workers, "softwareAgent" => $platformField, "workerunits" => $workerunit, "filtered" => $filteredField, "batches" => $batch, "children" => $children];
         $unit->update();
         $avg_clarity = \MongoDB\Entity::where('metrics.units.withoutSpam.' . $unit->_id, 'exists', 'true')->avg('metrics.units.withoutSpam.' . $unit->id . '.max_relation_Cos.avg');
         if (!isset($avg_clarity)) {
             $avg_clarity = 0;
         }
         $unit->avg_clarity = $avg_clarity;
         $unit->update();
         \Log::debug("Updated unit {$unit->_id}.");
     }
     $job->delete();
     // the Queue job...
 }
 public function getUnitsrelex()
 {
     $count = 0;
     foreach (\Workerunit::where('type', 'RelEx')->get() as $ann) {
         set_time_limit(30);
         if (!isset($ann->content)) {
             echo "{$ann->_id} no content\r\n";
             echo "--------------------------------\r\n";
             continue;
         }
         if (!empty($ann->unit_id)) {
             echo "{$ann->_id} has unitid\r\n";
             continue;
         }
         //dd($ann->question);
         //$xml = simplexml_load_string($ann->question);
         //$url = (string) $xml->ExternalURL;
         $xml = simplexml_load_string($ann->question);
         //$html = $ann->question;//
         $html = (string) $xml->HTMLContent;
         //dd($html);
         $dom = HtmlDomParser::str_get_html($html);
         $sentence = rtrim($dom->find('span[class=senval]', 0)->innertext, '.');
         $term1 = $dom->find('span[style=color:#0000CD;]', 1)->innertext;
         $term2 = $dom->find('span[style=color:#0000CD;]', 2)->innertext;
         /*			$sentence = "Poisson regression analysis which included data for multiple measurements of Tme/[TE] over the first year of life and adjusted for age-at-test and maternal smoking during [PREGNANCY] also demonstrated a greater decrease in Tme/Te in female infants who subsequently develop an LRI (P = 0.08";
         			$term1 = "[PREGNANCY]";
         			$term2 = "[TE]";*/
         $unit = \MongoDB\Entity::where('content.terms.first.formatted', $term1)->where('content.terms.second.formatted', $term2)->where('content.sentence.formatted', $sentence)->first();
         /*			if(!$unit){
         				$hi = 0;
         				$units = \MongoDB\Entity::where('content.terms.first.formatted', $term1)
         						->where('content.terms.second.formatted', $term2)
         						->get();
         
         				foreach($units as $punit){
         					try{
         						$pct = similar_text($sentence, $punit->content['sentence']['formatted']);
         					} catch (ErrorException $e) {
         						echo "\r\n\r\n\r\n\r\n{$punit->_id}\r\n\r\n\r\n\r\n";
         						$pct = similar_text(strtolower($sentence), strtolower($punit->content['sentence']['text']));
         					}	
         					if($pct>$hi) {
         						$hi=$pct;
         						$unit = $punit;
         					}
         				}
         
         			} */
         if ($unit) {
             echo "\r\n\\YY {$sentence}\r\n";
             echo "== " . $unit->content['sentence']['formatted'];
             echo "\r\n{$ann->_id}->{$unit->_id}\r\n";
             $ann->unit_id = $unit->_id;
             $ann->save();
         } else {
             echo "\r\nNO {$ann->_id}\r\n{$term1}--{$term2}--{$sentence}\r\n";
             //echo $punit->content['sentence']['formatted'];
             //echo "\r\n{$ann->unit_id}----------------\r\n";
             echo "----------------------------------------";
             continue;
         }
     }
 }
Example #24
0
 public function OldcreateWorkerunitsAndCrowdAgents($mappedWorkerunitsWithUnits, $job_id, $taskType = "FactSpan")
 {
     $status = array();
     $index = 0;
     try {
         $activity = new Activity();
         $activity->softwareAgent_id = "cf";
         $activity->save();
     } catch (Exception $e) {
         $activity->forceDelete();
         $status['error'][$index]['activity'] = $e->getMessage();
     }
     foreach ($mappedWorkerunitsWithUnits as $mappedWorkerunitsWithUnit) {
         $index++;
         $crowdagent = CrowdAgent::where('platformAgentId', $mappedWorkerunitsWithUnit['_worker_id'])->where('softwareAgent_id', 'cf')->first();
         if (!$crowdagent) {
             try {
                 $crowdagent = new CrowdAgent();
                 $crowdagent->_id = "crowdagent/cf/" . $mappedWorkerunitsWithUnit['_worker_id'];
                 $crowdagent->softwareAgent_id = 'cf';
                 $crowdagent->platformAgentId = (int) $mappedWorkerunitsWithUnit['_worker_id'];
                 $crowdagent->country = $mappedWorkerunitsWithUnit['_country'];
                 $crowdagent->region = $mappedWorkerunitsWithUnit['_region'];
                 $crowdagent->city = $mappedWorkerunitsWithUnit['_city'];
                 $crowdagent->cfWorkerTrust = (double) $mappedWorkerunitsWithUnit['_trust'];
                 $crowdagent->save();
             } catch (Exception $e) {
                 $status['error'][$index]['crowdagent'] = $e->getMessage();
                 // continue;
             }
         }
         if (!Entity::where('softwareAgent_id', 'cf')->where('platformWorkerunitId', $mappedWorkerunitsWithUnit['_id'])->first()) {
             $entity = new Entity();
             $entity->format = "text";
             $entity->domain = "medical";
             $entity->documentType = "workerunit";
             $entity->job_id = $job_id;
             $entity->activity_id = $activity->_id;
             $entity->crowdAgent_id = $crowdagent->_id;
             $entity->softwareAgent_id = "cf";
             $entity->unit_id = $mappedWorkerunitsWithUnit['unit']['_id'];
             $entity->platformWorkerunitId = (int) $mappedWorkerunitsWithUnit['_id'];
             $entity->cfChannel = $mappedWorkerunitsWithUnit['_channel'];
             $entity->acceptTime = new MongoDate(strtotime($mappedWorkerunitsWithUnit['_started_at']));
             $entity->submitTime = new MongoDate(strtotime($mappedWorkerunitsWithUnit['_created_at']));
             $entity->cfTrust = (double) $mappedWorkerunitsWithUnit['_trust'];
             if ($taskType == "FactSpan") {
                 $entity->content = ["confirmfirstfactor" => $mappedWorkerunitsWithUnit['confirmfirstfactor'], "confirmsecondfactor" => $mappedWorkerunitsWithUnit['confirmsecondfactor'], "firstfactor" => $mappedWorkerunitsWithUnit['firstfactor'], "secondfactor" => $mappedWorkerunitsWithUnit['secondfactor'], "saveselectionids1" => $mappedWorkerunitsWithUnit['saveselectionids1'], "saveselectionids2" => $mappedWorkerunitsWithUnit['saveselectionids2'], "confirmids1" => $mappedWorkerunitsWithUnit['confirmids1'], "confirmids2" => $mappedWorkerunitsWithUnit['confirmids2'], "sentencefirstfactor" => $mappedWorkerunitsWithUnit['sentencefirstfactor'], "sentencesecondfactor" => $mappedWorkerunitsWithUnit['sentencesecondfactor']];
             } elseif ($taskType == "RelEx") {
                 $entity->content = ["step_1_select_the_valid_relations" => $mappedWorkerunitsWithUnit['step_1_select_the_valid_relations'], "step_2a_copy__paste_only_the_words_from_the_sentence_that_express_the_relation_you_selected_in_step1" => $mappedWorkerunitsWithUnit['step_2a_copy__paste_only_the_words_from_the_sentence_that_express_the_relation_you_selected_in_step1'], "step_2b_if_you_selected_none_in_step_1_explain_why" => $mappedWorkerunitsWithUnit['step_2b_if_you_selected_none_in_step_1_explain_why']];
             } elseif ($taskType == "RelDir") {
                 $entity->content = ["direction" => $mappedWorkerunitsWithUnit['direction']];
             }
             try {
                 $entity->save();
             } catch (Exception $e) {
                 $status['error'][$index]['entity'] = $e->getMessage();
             }
         }
     }
     return $status;
 }
Example #25
0
 /**
  * Display a listing of the resource.
  *
  * @return Response
  */
 public function index()
 {
     /**
      * Return one entity
      *
      * @return one Entity
      */
     if (Input::has('id')) {
         $id = Input::get('id');
         // Check if is workerunit, when workerunit append units, if not append workerunits (assumption that it is a unit so far valid)
         $workerunitType = strpos($id, 'workerunit');
         if ($workerunitType === false) {
             $result = \MongoDB\Entity::with('hasWorkerunits')->where('_id', $id)->get();
         } else {
             $result = \MongoDB\Entity::with('hasUnit')->where('_id', $id)->get();
         }
         $result = $result->toArray();
         return $result;
     }
     $documents = $this->repository->returnCollectionObjectFor("entity")->where('documentType', 'job')->with('hasConfiguration')->with('wasAttributedToUserAgent');
     //Filter on wished for fields using using field of v2
     json_encode($documents);
     if (Input::has('filter')) {
         foreach (Input::get('filter') as $filter => $value) {
             if (is_numeric($value)) {
                 $documents = $documents->where($filter, (int) $value);
                 continue;
             }
             if ($filter == "userAgent") {
                 $filter = "user_id";
             }
             if (is_array($value)) {
                 foreach ($value as $operator => $subvalue) {
                     if ($filter == "username") {
                         $documents = $documents->where('user_id', 'like', '%' . $subvalue . '%');
                         continue;
                     }
                     if (in_array($operator, $this->operators)) {
                         if (is_numeric($subvalue)) {
                             $subvalue = (int) $subvalue;
                         }
                         if ($operator == 'like') {
                             $subvalue = '%' . $subvalue . '%';
                         }
                         // if (strpos($a,'are') !== false) {
                         // 	   echo 'true';
                         // }
                         if (strpos($filter, "hasConfiguration") !== false) {
                             $filter = explode(".", $filter);
                             $jobConf = Entity::where('documentType', '=', 'jobconf')->where(end($filter), 'like', $subvalue);
                             $allJobConfIDs = array_flatten($jobConf->get(['_id'])->toArray());
                             $documents = $documents->whereIn('jobConf_id', $allJobConfIDs);
                             continue;
                         }
                         $documents = $documents->where($filter, $operator, $subvalue);
                     }
                 }
                 continue;
             } else {
                 $value = array($value);
             }
             $documents = $documents->whereIn($filter, $value);
         }
     }
     // Make sort possible on
     if (Input::has('sortBy')) {
         $sortBy = Input::get('sortBy');
         if (Input::has('order')) {
             $order = Input::get('order');
         }
         $documents = $documents->OrderBy($sortBy, $order);
     }
     // If no sort is selected, newest jobs come on top
     if (!Input::has('sortBy')) {
         $documents = $documents->OrderBy('created_at', 'des');
     }
     // Take limit of 100 unless otherwise indicated
     // if(!$limit = (int) Input::get('limit'))
     // {
     // 	$limit = 100;
     // }
     if (!($perPage = (int) Input::get('perPage'))) {
         $perPage = 10;
     }
     if (!($page = (int) Input::get('page'))) {
         $page = 1;
         $calcPage = 0;
     } else {
         $calcPage = $page - 1;
     }
     $total = $documents->count();
     $skip = $calcPage * $perPage;
     $jobs = $documents->skip($skip)->take($perPage)->get();
     // Paginate results, total amount of records, records per page and currentPage etc.
     //todo efficiency improvement to cache $jobs for pagination
     $paginator = array("total" => $total, "perPage" => $perPage, "currentPage" => $page, "data" => $jobs->toArray());
     //dd($paginator);
     return Response::json($paginator);
     // Take limit of 100 unless otherwise indicated
     //Eager load jobConfiguration into job entity
     $entities = $documents->take($limit)->get();
     $jobs = array();
     //Push entity objects into array for paginator
     foreach ($entities as $entity) {
         array_push($jobs, $entity);
     }
     // Paginate results, current page, page of choice etc.
     if (!($perPage = (int) Input::get('perpage'))) {
         $perPage = 2;
     }
     $paginator = Paginator::make($jobs, count($entities), $perPage);
     //Return paginator
     return Response::json($paginator);
 }