/**
*
* @param array $input
* @return result status structure containing
* 'status' 'ok' or 'error'
* 'message' 'A status message'
* 'batch' the batch or null if error
*/
public function store(array $input)
{
try {
$this->createBatchCreatorSoftwareAgent();
} catch (Exception $e) {
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "batchcreator";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
try {
$entity = new Entity();
$entity->title = $input['batch_title'];
$entity->format = $input['format'];
$entity->domain = $input['domain'];
$entity->documentType = 'batch';
$entity->softwareAgent_id = 'batchcreator';
$entity->parents = $input['units'];
$entity->content = $input['batch_description'];
$entity->hash = md5(serialize($entity->parents));
$entity->activity_id = $activity->_id;
$entity->save();
Queue::push('Queues\\UpdateUnits', $input['units']);
return ['status' => 'ok', 'message' => $input['batch_title'] . " batch was successfully created. (URI: {$entity->_id})", 'batch' => $entity];
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
return ['status' => 'error', 'message' => $e->getMessage(), 'batch' => null];
}
}
public function workerunit_output_cf_factorspan_merged_relex($mappedWorkerunitsWithUnits)
{
$tempEntityID = null;
$status = array();
try {
$activity = new Activity();
$activity->softwareAgent_id = "importer";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error']['activity'] = $e->getMessage();
return $status;
}
$relexStructurer = App::make('\\preprocess\\RelexStructurer');
$test = array();
foreach ($mappedWorkerunitsWithUnits as $mappedWorkerunitsWithUnitKey => $mappedWorkerunitsWithUnitVal) {
$parentEntity = $mappedWorkerunitsWithUnitVal['unit'];
$parentEntity['content']['terms']['first']['text'] = strtolower(str_replace(["[", "]"], ["", ""], $mappedWorkerunitsWithUnitVal['term1']));
$parentEntity['content']['terms']['second']['text'] = strtolower(str_replace(["[", "]"], ["", ""], $mappedWorkerunitsWithUnitVal['term2']));
$parentEntity['content']['terms']['first']['formatted'] = $mappedWorkerunitsWithUnitVal['term1'];
$parentEntity['content']['terms']['first']['startIndex'] = (int) $mappedWorkerunitsWithUnitVal['b1'];
$parentEntity['content']['terms']['first']['endIndex'] = (int) $mappedWorkerunitsWithUnitVal['e1'];
$parentEntity['content']['terms']['second']['formatted'] = $mappedWorkerunitsWithUnitVal['term2'];
$parentEntity['content']['terms']['second']['startIndex'] = (int) $mappedWorkerunitsWithUnitVal['b2'];
$parentEntity['content']['terms']['second']['endIndex'] = (int) $mappedWorkerunitsWithUnitVal['e2'];
$parentEntity['content']['sentence']['formatted'] = $mappedWorkerunitsWithUnitVal['sentence'];
$parentEntity['content']['properties']['overlappingTerms'] = $relexStructurer->overlappingTerms($parentEntity['content']);
$title = $parentEntity['title'] . "_FS_" . $mappedWorkerunitsWithUnitKey;
try {
$entity = new Entity();
$entity->_id = $tempEntityID;
$entity->sent_id = $mappedWorkerunitsWithUnitVal['Sent_id'];
$entity->title = strtolower($title);
$entity->domain = $parentEntity['domain'];
$entity->format = $parentEntity['format'];
$entity->documentType = "relex-structured-sentence";
$entity->parents = array($parentEntity['_id']);
$entity->content = $parentEntity['content'];
unset($parentEntity['properties']);
$entity->hash = md5(serialize($parentEntity['content']));
$entity->activity_id = $activity->_id;
$entity->save();
array_push($test, $entity->toArray());
$status['success'][$title] = $title . " was successfully processed into a relex-structured-sentence. (URI: {$entity->_id} {$entity->hash})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$entity->forceDelete();
$status['error'][$title] = $e->getMessage() . " " . $entity->hash;
}
$tempEntityID = $entity->_id;
}
return $test;
return $status;
}
public function storeVideoDescription($parentEntity)
{
$title = "Description: " . $parentEntity->title;
$status = array();
try {
$this->createOpenimagesVideoDescriptionExtractorSoftwareAgent();
} catch (Exception $e) {
$status['error']['OnlineDataVideoDescr'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "videodescriptiongetter";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$status['error']['OnlineDataVideoDescr'] = $e->getMessage();
$activity->forceDelete();
return $status;
}
//dd($parentEntity->title);
$languageValues = $parentEntity->content["metadata"]["abstract"];
foreach ($languageValues as $lang => $value) {
if ($value != "" || $value != NULL) {
try {
$entity = new Entity();
$entity->_id = $entity->_id;
$entity->title = $lang . '.' . strtolower($title);
$entity->domain = $parentEntity->domain;
$entity->format = "text";
$entity->documentType = "metadatadescription";
$entity->source = "openimages";
$entity->videoContent = $parentEntity->content["storage_url"];
$entity->videoTitle = $parentEntity->content["metadata"]["title"][$lang];
$entity->language = $lang;
$entity->parents = array($parentEntity->_id);
$content = array();
$content["description"] = $value;
$entity->content = $content;
$preprocessed = array();
$preprocessed["automatedEntities"] = false;
$preprocessed["automatedEvents"] = false;
$entity->preprocessed = $preprocessed;
$entity->tags = array("unit");
$entity->hash = md5(serialize([$entity->content]));
$entity->activity_id = $activity->_id;
$entity->save();
Queue::push('Queues\\UpdateUnits', [$entity->_id]);
$status['success'][$title] = $title . " was successfully uploaded. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
}
}
return $status;
}
public function postFeatures()
{
$return = array('status' => 'ok');
$input = Input::get();
$domain = $input[1];
$type = $input[2];
// CREATE ACTIVITY FOR BATCH
$activity = new Activity();
$activity->label = "Images posted for processing.";
$activity->softwareAgent_id = 'imagegetter';
$activity->save();
// LOOP THROUGH IMAGES CREATE ENTITIES WITH ACTIVITY-ID FOR NEW IMAGES
$url_ids = "";
foreach ($input[0] as $img) {
\Log::debug(json_encode($img));
try {
$parse = parse_url($img['url']);
//$source = $parse['host'];
// Save images as parent
$image = new Entity();
$image->domain = $domain;
$image->format = "image";
$content = $image->content;
$content['url'] = $img['url'];
$content['title'] = $img['title'];
$content['height'] = $img['height'];
$content['width'] = $img['width'];
$content['description'] = $img['description'];
$content['author'] = $img['author'];
$image->content = $content;
$image->documentType = $type;
$image->source = "Rijksmuseum";
$image->tags = ['unit'];
$image->activity_id = $activity->_id;
$image->softwareAgent_id = "imagegetter";
// Take last part of URL as image title
$temp = explode('/', $img['url']);
//$image->title = end($temp);
// CHECK WHETHER URL EXISTS ALREADY
$hash = md5(serialize($image->content));
if ($existingid = Entity::where('hash', $hash)->pluck('_id')) {
$imageid = $existingid;
} else {
$image->hash = $hash;
$image->activity_id = $activity->_id;
\Log::debug(json_encode($image->toArray()));
$image->save();
$existingid = $image->_id;
}
$url_ids .= "{$img['url']} {$existingid} ";
} catch (Exception $e) {
//delete image
if (isset($image)) {
$image->forceDelete();
}
//delete activity
if (isset($activity)) {
$activity->forceDelete();
}
//Session::flash('flashError', $e->getMessage());
$return['error'] = $e->getMessage();
$return['status'] = 'bad';
\Log::debug($e->getMessage());
return $return;
}
// RUN PYTHON SCRIPT THAT CALLS APIs TO ADD FEATURES TO IMAGE
}
//return $url_ids;
try {
//$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getRijks.py " . $domain . " " . $type . " " . 4 . " " . "vogel";
$command = "/usr/bin/python2.7 " . base_path() . "/app/lib/getAPIS/getMany.py " . $domain . " " . $type . " " . Auth::user()->email . " " . $url_ids;
//$command = "/usr/bin/python2.7 /var/www/crowd-watson/app/lib/getAPIS/getMany.py art painting http://lh3.ggpht.com/Q1GZTdmwa8iTLgdbu5uAgzovmLbb7lsYhG-QgVcoN8A-WJtIsNUo4-VyTMd9iKHLp-XNm812WyUaSgQdHdjQjDioJQI=s0 999";
//return $command;
\Log::debug("Running {$command}");
exec($command, $output, $error);
$return['oo'] = $output;
$return['ee'] = $error;
//$return['a'] = $a;
//throw $e; // for debugging.
//return $error;
} catch (Exception $e) {
//throw $e; // for debugging.
\Log::debug("ERROR: " . $e->getMessage());
$return['error'] = $e->getMessage();
$return['status'] = 'bad';
}
return $this->returnJson($return);
}
public function storeVideoSegments($parentEntity, $videoSegmenting)
{
$tempEntityID = null;
$status = array();
try {
$this->createVideoSegmentingSoftwareAgent();
} catch (Exception $e) {
$status['error']['videosegmenting'] = $e->getMessage();
return $status;
}
try {
$activity = new Activity();
$activity->softwareAgent_id = "videosegmenting";
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error'][$title] = $e->getMessage();
return $status;
}
for ($i = 0; $i < sizeof($videoSegmenting); $i++) {
$videoSegmentName = explode("/", $videoSegmenting[$i]["storage_url"]);
$title = $videoSegmentName[sizeof($videoSegmentName) - 1];
try {
$entity = new Entity();
$entity->_id = $tempEntityID;
$entity->title = strtolower($title);
$entity->domain = $parentEntity->domain;
$entity->format = "video";
$entity->documentType = "videosegment";
$entity->parents = array($parentEntity->_id);
$entity->source = $parentEntity->source;
$entity->content = $videoSegmenting[$i];
//unset($relexStructuredSentenceKeyVal['properties']);
$entity->hash = md5(serialize($videoSegmenting[$i]));
$entity->activity_id = $activity->_id;
$entity->save();
$status['success'][$title] = $title . " was successfully processed into a video segment. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
$tempEntityID = $entity->_id;
}
$status['success']['noEntitiesCreated'] = sizeof($videoSegmenting);
//dd($status);
return $status;
}
/**
* Store a new file to the database. Construct all entity information for such file.
*
* @param $fileFormat
* @param $domain
* @param $documentType
* @param $project The name of the Project who owns the file data.
* @param $domainCreate
* @param $documentCreate
* @param $files
*/
public function store($fileFormat, $domain, $documentType, $project, $domainCreate, $documentCreate, $files)
{
$format = $this->getType($fileFormat);
$validatedFiles = $this->performValidation($files, $format);
$newDomain = false;
$newDocType = false;
if ($domain == 'domain_type_other') {
// Add new domain to DB
$domain = $domainCreate;
$domain = str_replace(' ', '', $domain);
$domain = strtolower($domain);
$domain = 'domain_type_' . $domain;
$newDomain = true;
}
if ($documentType == 'document_type_other') {
// Add new doc_type to DB
$documentType = $documentCreate;
$newDocType;
}
if ($newDomain || $newDocType) {
if ($newDomain) {
// newDomain and new DocType
$domainName = $domainCreate;
$upDomains = $this->softwareComponent->domains;
$upDomains[$domain] = ["name" => $domainName, "file_formats" => [$fileFormat], "document_types" => [$documentType]];
$this->softwareComponent->domains = $upDomains;
} else {
if ($newDocType) {
// Only docType is new -- domain already existed...
$docTypes = $this->softwareComponent->domains[$domain]["document_types"];
array_push($docTypes, $documentType);
$this->softwareComponent->domains[$domain]["document_types"] = $docTypes;
}
}
$this->softwareComponent->save();
}
$domain = str_replace("domain_type_", "", $domain);
$documentType = str_replace("document_type_", "", $documentType);
$status = [];
try {
$activity = new Activity();
$activity->softwareAgent_id = $this->softwareComponent->_id;
$activity->save();
} catch (Exception $e) {
// Something went wrong with creating the Activity
$activity->forceDelete();
$status['error'] = $e->getMessage();
return $status;
}
$files = $validatedFiles['passed'];
foreach ($files as $file) {
$title = $file->getClientOriginalName();
try {
$entity = new Entity();
$entity->_id = $entity->_id;
$entity->activity_id = $activity->_id;
$entity->softwareAgent_id = $this->softwareComponent->_id;
$entity->project = $project;
$entity->title = strtolower($title);
$entity->domain = $domain;
$entity->format = "text";
$entity->documentType = $documentType;
$entity->content = File::get($file->getRealPath());
$entity->hash = md5(serialize([$entity->content]));
$entity->tags = ["unit"];
$entity->save();
$status['success'][$title] = $title . " was successfully uploaded. (URI: {$entity->_id})";
} catch (Exception $e) {
// Something went wrong with creating the Entity
$activity->forceDelete();
$entity->forceDelete();
$status['error'][$title] = $e->getMessage();
}
}
$files = $validatedFiles['failed'];
foreach ($files as $file) {
$title = $file->getClientOriginalName();
$status['error'][$title] = 'Validation failed';
}
return $status;
}
