/**
* Show the form for editing the specified resource.
*
* @param int $id
* @return Response
*/
public function edit(Dataset $dataset)
{
$datasources = Datasource::lists('name', 'id');
$categories = DatasetCategory::all()->lists('name', 'id');
$subcategories = DatasetSubcategory::all()->lists('name', 'id');
return view('datasets.edit', compact('dataset', 'categories', 'subcategories', 'datasources'));
}
/**
* Remove the specified resource from storage.
*
* @param int $id
* @return Response
*/
public function destroy(Datasource $datasource, Request $request)
{
//check depedencies first
if ($datasource->datasets()->count() > 0) {
return redirect()->route('datasources.index')->with('message', 'Some datasets are linked to this source, so you cannot delete it.')->with('message-class', 'error');
}
if ($datasource->variables()->count() > 0) {
return redirect()->route('datasources.index')->with('message', 'Some variables are linked to this source, so you cannot delete it.')->with('message-class', 'error');
}
if ($datasource->values()->count() > 0) {
return redirect()->route('datasources.index')->with('message', 'Some values are linked to this source, so you cannot delete it.')->with('message-class', 'error');
}
//no dependencies, delete
$datasource->delete();
Cache::flush();
return redirect()->route('datasources.index')->with('message', 'Source deleted.');
}
/**
* Import a collection of variables
*/
public function variables(Request $request)
{
$input = $request->all();
return DB::transaction(function () use($input) {
// First, we create the dataset object itself
$dataset = $input['dataset'];
// entityKey is a unique list of entity names/codes e.g. ['Germany', 'Afghanistan', 'USA']
$entityKey = $input['entityKey'];
// entities is a list of indices for entityKey
$entities = $input['entities'];
$years = $input['years'];
$variables = $input['variables'];
if (isset($dataset['sourceId'])) {
$sourceId = $dataset['sourceId'];
} else {
$source = $input['source'];
$sourceProps = ['name' => $source['name'], 'description' => $source['description']];
$sourceId = Datasource::create($sourceProps)->id;
}
if (isset($dataset['id'])) {
$datasetId = $dataset['id'];
} else {
$datasetProps = ['name' => $dataset['name'], 'description' => isset($dataset['description']) ? $dataset['description'] : "", 'fk_dst_cat_id' => $dataset['categoryId'], 'fk_dst_subcat_id' => $dataset['subcategoryId'], 'fk_dsr_id' => $sourceId];
$datasetId = Dataset::create($datasetProps)->id;
}
// First, we insert all of the entities with "on duplicate key update", ensuring
// they all exist in the database.
$insertQuery = "INSERT INTO entities (name, fk_ent_t_id) VALUES";
$pdo = DB::connection()->getPdo();
foreach ($entityKey as $name) {
if ($name != $entityKey[0]) {
$insertQuery .= ",";
}
$insertQuery .= " (" . $pdo->quote($name) . ",5)";
}
$insertQuery .= " ON DUPLICATE KEY UPDATE name=VALUES(name);";
DB::statement($insertQuery);
// Now we need to pull them back out again so we know what ids to go with what names
$entityNameToId = DB::table('entities')->select('id', 'name')->whereIn('name', $entityKey)->lists('id', 'name');
// Now we feed in our set of variables and associated data
foreach ($variables as $variable) {
$values = $variable['values'];
$newVariable = ['name' => $variable['name'], 'description' => $variable['description'], 'unit' => $variable['unit'], 'fk_var_type_id' => $variable['typeId'], 'fk_dst_id' => $datasetId, 'fk_dsr_id' => $sourceId, 'uploaded_by' => \Auth::user()->name, 'uploaded_at' => Carbon::now()];
$varId = DB::table('variables')->insertGetId($newVariable);
$newDataValues = [];
for ($i = 0; $i < sizeof($years); $i++) {
if ($values[$i] === '') {
// Empty cells, as opposed to zeroes, most likely should not be inserted at all
continue;
}
$newDataValues[] = ['fk_var_id' => $varId, 'fk_ent_id' => $entityNameToId[$entityKey[$entities[$i]]], 'year' => $years[$i], 'value' => $values[$i]];
// For very big datasets, there may be too many to do in a single insert
// Limit is about 25565 placeholders. So we stop and push in a bunch every so often
if (sizeof($newDataValues) > 10000) {
DB::table('data_values')->insert($newDataValues);
$newDataValues = [];
}
}
if (sizeof($newDataValues) > 0) {
DB::table('data_values')->insert($newDataValues);
}
}
return ['datasetId' => $datasetId];
});
}
/**
* Show the form for editing the specified resource.
*
* @param int $id
* @return Response
*/
public function edit(Variable $variable)
{
$datasources = Datasource::lists('name', 'id');
return view('variables.edit', compact('variable', 'datasources'));
}
/**
* Show the form for editing the specified resource.
*
* @param int $id
* @return Response
*/
public function edit(DataValue $dataValue)
{
$entities = Entity::lists('name', 'id');
$datasources = Datasource::lists('name', 'id');
return view('values.edit', compact('dataValue', 'entities', 'datasources'));
}
/**
* Execute the command.
*
* @return void
*/
public function handle()
{
//set_time_limit( 600 );
//ini_set('memory_limit', '256M');
//bump up limits
set_time_limit(600);
ini_set('memory_limit', '256M');
return;
//will we be checking entities
$entityCheck = $this->validate_entities == 'on' ? false : true;
//create new file
$inputFileData = ['raw_data' => $this->data, 'fk_user_id' => $this->userId];
$inputFile = InputFile::create($inputFileData);
$inputFileDataId = $inputFile->id;
$multivariantDataset = $this->multivariant_dataset;
$variables = $this->variables;
if (!empty($variables)) {
$entityData = [];
//creating new datasource, if there is some
$sourceName = $this->source_name;
if (!empty($sourceName)) {
$datasourceData = ['name' => $this->source_name, 'link' => $this->source_link, 'description' => $this->source_description];
$datasource = Datasource::create($datasourceData);
} else {
//fake datasoure
$datasource = new \stdClass();
$datasource->id = null;
}
//create new dataset or pick existing one
$datasetName = $this->new_dataset_name;
$datasetData = ['name' => $datasetName, 'fk_dst_cat_id' => $this->category_id, 'fk_dst_subcat_id' => $this->subcategory_id, 'description' => $this->new_dataset_description, 'fk_dsr_id' => $datasource->id];
$dataset = Dataset::create($datasetData);
$datasetId = $dataset->id;
//process possible tags
$tagsInput = $this->new_dataset_tags;
if (!empty($tagsInput)) {
$tagsArr = explode(',', $tagsInput);
foreach ($tagsArr as $tag) {
$tag = DatasetTag::create(['name' => $tag]);
$tagId = $tag->id;
$datasetTagLink = LinkDatasetsTags::create(['fk_dst_id' => $datasetId, 'fk_dst_tags_id' => $tagId]);
}
}
//store inserted variables, for case of rolling back
$inserted_variables = array();
foreach ($variables as $variableJsonString) {
//convert back single out to actual single quote
//$variableJsonString = str_replace( "'", "‘", $variableJsonString );
//setting json_decode second param to false, to try to save memory
$variableObj = json_decode($variableJsonString, false);
$variableData = ['name' => $variableObj->name, 'fk_var_type_id' => $this->variable_type, 'fk_dst_id' => $datasetId, 'unit' => $variableObj->unit, 'description' => $variableObj->description, 'fk_dsr_id' => $datasource->id];
//update of existing variable or new variable
if (!isset($variableObj->id)) {
//new variable
$variable = Variable::create($variableData);
} else {
//update variable
$variable = Variable::find($variableObj->id);
$variable->fill($variableData);
$variable->save();
}
$variableId = $variable->id;
$inserted_variables[] = $variable;
$variableValues = $variableObj->values;
foreach ($variableValues as $countryValue) {
$entityData = ['name' => $countryValue->key, 'fk_ent_t_id' => 5, 'validated' => 0];
if ($entityCheck) {
//entity validation (only if not multivariant dataset)
//find corresponding iso code
$entityIsoName = EntityIsoName::match($entityData['name'])->first();
if (!$entityIsoName) {
//!haven't found corresponding country, throw an error!
//rollback everything first
foreach ($inserted_variables as $inserted_var) {
$inserted_var->data()->delete();
$inserted_var->delete();
}
//is new dataset
if ($this->new_dataset === '1') {
$dataset = Dataset::find($datasetId);
//delete itself
$dataset->delete();
}
\Log::error('Error non-existing entity in dataset.');
\Log::error($entityData['name']);
return redirect()->route('import')->with('message', 'Error non-existing entity in dataset.')->with('message-class', 'error');
}
//enter standardized info
$entityData['name'] = $entityIsoName->name;
$entityData['code'] = $entityIsoName->code;
$entityData['validated'] = 1;
}
//find try finding entity in db
if (isset($entityIsoName)) {
$entity = Entity::where('code', $entityIsoName->code)->first();
} else {
//not standardized data
$entity = Entity::where('code', $entityData['name'])->orWhere('name', $entityData['name'])->first();
}
if (!$entity) {
//entity haven't found in database, so insert it
$entity = Entity::create($entityData);
}
//check to override validation if stored in db not validated and now is validate
if ($entity->validated == 0 && $entityData['validated'] === 1) {
$entity->validated = 1;
$entity->save();
}
$entityId = $entity->id;
$countryValues = $countryValue->values;
//prepare vars for mass insert
$times = [];
$values = [];
//TODO - get latest time for base timeId
$lastTime = Time::orderBy('id', 'desc')->first();
$timeId = !empty($lastTime) ? $lastTime->id : 0;
foreach ($countryValues as $value) {
if ($this->hasValue($value->x) && $this->hasValue($value->y)) {
$timeId++;
//create time
$timeObj = $value->x;
$timeValue = ['startDate' => isset($timeObj->sd) ? $timeObj->sd : "", 'endDate' => isset($timeObj->ed) ? $timeObj->ed : "", 'date' => isset($timeObj->d) ? $timeObj->d : "", 'label' => isset($timeObj->l) ? $timeObj->l : ""];
//convert timedomain
$fk_ttype_id = 1;
if (!empty($timeObj->td)) {
$ttQuery = TimeType::query();
$fk_ttype_id = $ttQuery->whereRaw('LOWER(`name`) like ?', [$timeObj->td])->first()->id;
}
$timeValue['fk_ttype_id'] = $fk_ttype_id;
//using mass insert instead
//$time = Time::create( $timeValue );
//$timeId = $time->id;
$times[] = $timeValue;
//create value
$dataValueData = ['value' => $value->y, 'fk_time_id' => $timeId, 'fk_input_files_id' => $inputFileDataId, 'fk_var_id' => $variableId, 'fk_ent_id' => $entityId, 'fk_dsr_id' => $datasource->id];
//using mass insert instead
//$dataValue = DataValue::create( $dataValueData );
$values[] = $dataValueData;
}
}
//mass insertion
Time::insert($times);
DataValue::insert($values);
}
}
}
}
public function datasource(Request $request)
{
try {
$sourceName = $request->has('name') ? $request->get('name') : '';
$sourceLink = $request->has('link') ? $request->get('link') : '';
$sourceDescription = $request->has('description') ? $request->get('description') : '';
if (!empty($sourceName)) {
$datasourceData = ['name' => $sourceName, 'link' => $sourceLink, 'description' => $sourceDescription];
$datasource = Datasource::create($datasourceData);
$datasourceId = $datasource->id;
return ['success' => true, 'data' => ['datasourceId' => $datasourceId]];
}
return ['success' => false];
} catch (Exception $e) {
return ['success' => false];
}
}