/** * @param $json text content of asset data file */ private function reload($json) { $ss = new SchemaService(); try { $resp = $ss->isValid($json); if (is_array($resp)) { $data = Util::decode_json($json); static::$logger->error(__CLASS__ . '::' . __FUNCTION__ . ': schema in use: ' . $data->mimeType); foreach ($resp as $k => $v) { foreach ($v as $item => $line) { static::$logger->error(__CLASS__ . '::' . __FUNCTION__ . ': data not validated: ' . $k . ': ' . $item . ' => ' . $line); } } throw new ReposeJsonInstantiationError(json_last_error_msg()); } else { $data = Util::decode_json($json); $this->loadData($data); static::$logger->addInfo(__CLASS__ . '::' . __FUNCTION__ . ': loaded json data'); } } catch (ReposeJsonInstantiationError $e) { static::$logger->addAlert(__CLASS__ . '::' . __FUNCTION__ . ': Failed to validate sidecar file: ' . $e->getMessage()); } }
private function ingestImageAsset($filepath, $type, $ffprobe_config) { $msg = "Creating probe, for file: {$filepath}"; $this->report[] = $msg; static::$logger->info($msg); try { $probe = FFProbe::create($ffprobe_config, static::$logger); if (!$probe) { throw new \Exception('FFProbe failed to create.'); } else { $msg = "Probe created"; $this->report[] = $msg; static::$logger->info($msg); } } catch (Exception $e) { $msg = "Probe creation failed: " . $e->getMessage(); $this->report[] = $msg; static::$logger->info($msg); } $assignedProperties = []; $msg = "Probing file properties: {$filepath}"; $this->report[] = $msg; static::$logger->info($msg); try { $props = $probe->format($filepath)->all(); if ($props) { $assignedProperties[] = $this->formatToJsonArray($props); } } catch (ExecutionFailureException $e) { $msg = $e->getMessage(); static::$logger->addWarning($msg); return false; // triggers rejection in ingest } $msg = "Probing exif properties"; $this->report[] = $msg; static::$logger->info($msg); $exif = new Exif($filepath); $msg = "Encoding derived metadata"; $this->report[] = $msg; static::$logger->info($msg); $exif_json = json_encode($exif->toArray(), JSON_UNESCAPED_SLASHES); $exif_obj = Util::decode_json($exif_json); $msg = "Encoding assignedProperties"; $this->report[] = $msg; static::$logger->info($msg); $json_props = implode(",\n", $assignedProperties); $data = Util::decode_json($json_props); if ($exif_obj) { $msg = "Attaching derived exif props"; $this->report[] = $msg; static::$logger->info($msg); $data->{'exif'} = $exif_obj; } $msg = "Returning derived data object"; $this->report[] = $msg; static::$logger->info($msg); return $data; }
/** * @param $data an asset instance sidecar document * @return bool * @throws \RS\Repose\Core\Exceptions\ReposeValidationException */ public function isValid($data) { // instantiate stdClass object $data = Util::decode_json($data); // find matching schema or use default $schema = $this->schema($data->mimeType); $schema = Util::decode_json($schema->toJson()); // make live instance, stdClass object $validator = new Validator(); $validator->check($data, $schema); return $validator->isValid() ?: $validator->getErrors(); }
/** * @param $uuid * @param $collection_name target directory (will create if new) * @param $volume_name target volume (will create if new) * @param string $disk_name default only (current support level) * @return array ['json'=>'copied|updated|failed', 'asset'=>'copied|failed'] */ public function copyAsset($uuid, $collection_name, $volume_name, $disk_name = 'default') { static::$logger->info(__CLASS__ . '::' . __FUNCTION__); $asset = static::$disk->asset($uuid); $src_path = $asset->parent()->parent()->parent()->protocol() . $asset->parent()->parent()->name() . DS . $asset->parent()->name() . DS . $uuid; static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': src_path: ' . $src_path); $src_json = $src_path . '.json'; $src_binary = $src_path . '.' . $asset->extension(); // verify the volume named actually exists or create $volume = static::$disk->volume($volume_name, $create = 1); static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': volume: ' . $volume->name()); if (!static::$manager->has('default://' . $volume->name())) { static::$manager->createDir('default://' . $volume->name()); static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': created concrete volume: ' . $volume->name()); } // verify collection named actually exists or create $collection = $volume->collection($collection_name, null, $create = 1); static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': collection: ' . $collection->name()); if (!static::$manager->has('default://' . $volume->name() . DS . $collection->name())) { static::$manager->createDir('default://' . $volume->name() . DS . $collection->name()); static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': created concrete collection: ' . $collection->name()); } $new_uuid = Uuid::uuid4(); static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': created new uuid: ' . $new_uuid); // define target path (verifying volume/collection are valid in process) $target_path = static::$disk->protocol() . $volume->name() . DS . $collection->name() . DS . $new_uuid; static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': target_path: ' . $target_path); $target_json = $target_path . '.json'; $target_binary = $target_path . '.' . $asset->extension(); static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': target_json: ' . $target_json); static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': target_binary: ' . $target_binary); $results = []; if (!static::$manager->has($src_json)) { static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': no source file: ' . $src_json); } if ($ret = static::$manager->copy($src_json, $target_json)) { static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': copied json file'); $results['metadata'] = 'copied'; $json = Util::decode_json(static::$manager->read($target_json)); $json->uuid = $new_uuid; $json->parent = $collection->name(); $json->derivedProperties->{'copy_of'} = $uuid; $json->derivedProperties->{'copies'} = []; // there have been no copies made of this file yet $text = json_encode($json, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); if (static::$manager->put($target_json, $text)) { $results['metadata'] = 'updated'; } } else { static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': failed copying json file: ' . $ret); $results['metadata'] = 'failed'; } if (static::$manager->copy($src_binary, $target_binary)) { static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': copied binary'); $results['binary'] = 'copied'; } else { static::$logger->info(__CLASS__ . '::' . __FUNCTION__ . ': unable to copy binary'); $results['binary'] = 'failed'; } // TODO schemas need to include 'copies' and 'copy_of' fields // update teh src image's metadata with pointer to new copy $derivedProperties = $asset->derivedProperties(); static::$logger->debug(__CLASS__ . '::' . __FUNCTION__ . ': derivedProperties: ' . json_encode($derivedProperties, JSON_UNESCAPED_SLASHES)); if (!isset($derivedProperties->{'copies'})) { $derivedProperties->{'copies'} = []; } $derivedProperties->{'copies'}[] = $new_uuid; $asset->derivedProperties($derivedProperties); $json = $asset->toJson(); if (static::$manager->put($src_json, $json)) { $results['source'] = 'updated'; } else { $results['source'] = 'failed'; } return $results; }
public function ingest($type) { static::$logger->addInfo(__CLASS__ . '::' . __FUNCTION__ . ': staged file: ' . $this->staged_filename); // make relative to storage point $file = str_replace($this->staging . DS, '', $this->staged_filename); $file = 'default' . DS . $file; // working with disk level in src paths now - since 0.4.4 // snag collection name from src path (remove volume name if present) $this->collection_name = pathinfo($file, PATHINFO_DIRNAME); $this->collection_name = Util::trimVolumeName($this->collection_name); // if present ... // need the instance to associate with each asset before writing Json file $this->collection = static::$disk->collection($this->collection_name); // preserve for the record $originalName = pathinfo($file, PATHINFO_FILENAME); // without extension $originalExtension = pathinfo($file, PATHINFO_EXTENSION); // normalize $extension = Util::lower($originalExtension); // assign $uuid = Uuid::uuid4()->toString(); $target_filename = $uuid . '.' . $extension; $target_path = $this->collection_name; $target_spec = $target_path . DS . $target_filename; // alias for clarity - convenience // staged_filename has 'staging://' prefix // this does not want it $src_file = $file; if ($this->copyToTarget($src_file, $target_spec)) { static::$logger->addInfo($src_file . ' copied to ' . $target_spec); $this->removeFromStagingDir('staging://' . $src_file); $new_asset = new Asset(); $assetService = new AssetService($new_asset, ['path' => $target_path, 'filename' => $uuid]); // $info comes from Flysystem MountManager - filename is correct here $assetService->path($target_path); // want no protocol or file name here $new_asset->parent($assetService->getCollection()); // whether or not to use sidecar assignedProperties file from spool // or to save only the derived attributes from the ingest service // is there a specified sidecar assignedProperties file for this asset ? ie, in spool/staging // $originalName WITHOUT extension is key for md file, if any $md_key = $this->staging . DS . $this->collection_name . DS . $originalName; // findMetadataSidecarFile also validates the sidecar file $md_file_name = $this->findMetadataSidecarFile($md_key, $extension, $type); if (!is_null($md_file_name)) { $md = static::$manager->read('staging://' . $md_file_name); $new_asset->reload($md); } // set the basics from discovery $new_asset->uuid($uuid); $new_asset->parent($this->collection); $new_asset->mimeType($type); $new_asset->extension($extension); $new_asset->basename($originalName); $this->derivedProperties->{'created'} = time(); static::$logger->addInfo(__CLASS__ . '::' . __FUNCTION__ . ': derivedProps: ' . json_encode($this->derivedProperties, JSON_UNESCAPED_SLASHES)); // append file attributes as collected by ingest service $new_asset->derivedProperties($this->derivedProperties); // store all assignedProperties - derived and specified - in sidecar file $assetService->storeJsonData(); static::$logger->addInfo("Wrote new assignedProperties file: " . $assetService->jsonPath()); return true; } else { $this->reject($src_file); static::$logger->addAlert('Checksum verification failed: file: ' . $src_file . ' not copied to repository (now in rejects folder).'); } return false; }