Streamed, dynamically generated zip archives.
/**
* Stream a zip file to the client
*
* @param String $filename - Name for the file to be sent to the client
* @param Array $params - Optional parameters
* {
* expiration: '+10 minutes'
* }
*/
public function send($filename, $params = array())
{
// Set default values for the optional $params argument
if (!isset($params['expiration'])) {
$params['expiration'] = '+10 minutes';
}
// Initialize the ZipStream object and pass in the file name which
// will be what is sent in the content-disposition header.
// This is the name of the file which will be sent to the client.
$zip = new ZipStream\ZipStream($filename);
// Get a list of objects from the S3 bucket. The iterator is a high
// level abstration that will fetch ALL of the objects without having
// to manually loop over responses.
$result = $this->s3Client->getIterator('ListObjects', $this->params);
// We loop over each object from the ListObjects call.
foreach ($result as $file) {
// We need to use a command to get a request for the S3 object
// and then we can get the presigned URL.
$command = $this->s3Client->getCommand('GetObject', array('Bucket' => $this->params['Bucket'], 'Key' => $file['Key']));
$signedUrl = $command->createPresignedUrl($params['expiration']);
// Get the file name on S3 so we can save it to the zip file
// using the same name.
$fileName = basename($file['Key']);
// We want to fetch the file to a file pointer so we create it here
// and create a curl request and store the response into the file
// pointer.
// After we've fetched the file we add the file to the zip file using
// the file pointer and then we close the curl request and the file
// pointer.
// Closing the file pointer removes the file.
$fp = tmpfile();
$ch = curl_init($signedUrl);
curl_setopt($ch, CURLOPT_TIMEOUT, 120);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_exec($ch);
curl_close($ch);
$zip->addFileFromStream($fileName, $fp);
fclose($fp);
}
// Finalize the zip file.
$zip->finish();
}
private function buildFiles(\Project $project)
{
$translations = $project->getTranslations();
$builder = new KdybyTranslationBuilder();
$files = [];
foreach ($translations as $translation) {
$mask = '%s.' . $translation->getLocale() . '.neon';
$dictionaryData = $this->translationFacade->getDictionaryData($translation);
$outputFiles = $builder->build($mask, $dictionaryData);
$files = array_merge($files, $outputFiles);
}
$zip = new ZipStream(sprintf('%s.zip', $project->getName()));
foreach ($files as $fileName => $messages) {
$data = Neon::encode($messages, Neon::BLOCK);
$zip->addFile($fileName, $data);
}
$zip->finish();
}
public function handleDownloadTranslation()
{
$dictionaryData = $this->translationFacade->getDictionaryData($this->translation);
$builder = new KdybyTranslationBuilder();
$mask = '%s.' . $this->translation->getLocale() . '.neon';
$outputFiles = $builder->build($mask, $dictionaryData);
if (count($outputFiles) === 1) {
$name = key($outputFiles);
$neon = Neon::encode(current($outputFiles), Neon::BLOCK);
$response = new TextDownloadResponse($neon, $name, 'text/x-neon', 'UTF-8');
$this->sendResponse($response);
$this->terminate();
} else {
$zip = new ZipStream(sprintf('%s - %s.zip', $this->translation->getProject()->getName(), $this->translation->getLocale()));
foreach ($outputFiles as $fileName => $data) {
$neon = Neon::encode($data, Neon::BLOCK);
$zip->addFile($fileName, $neon);
}
$zip->finish();
}
}
public function testAddFileFromStream()
{
list($tmp, $stream) = $this->getTmpFileStream();
$zip = new ZipStream(null, array(ZipStream::OPTION_OUTPUT_STREAM => $stream));
$streamExample = fopen('php://temp', 'w+');
fwrite($streamExample, "Sample String Data");
rewind($streamExample);
// move the pointer back to the beginning of file.
$zip->addFileFromStream('sample.txt', $streamExample);
fclose($streamExample);
$streamExample2 = fopen('php://temp', 'w+');
fwrite($streamExample2, "More Simple Sample Data");
rewind($streamExample2);
// move the pointer back to the beginning of file.
$zip->addFileFromStream('test/sample.txt', $streamExample2);
fclose($streamExample2);
$zip->finish();
fclose($stream);
$tmpDir = $this->validateAndExtractZip($tmp);
$files = $this->getRecursiveFileList($tmpDir);
$this->assertEquals(array('sample.txt', 'test/sample.txt'), $files);
$this->assertEquals(file_get_contents($tmpDir . '/sample.txt'), 'Sample String Data');
$this->assertEquals(file_get_contents($tmpDir . '/test/sample.txt'), 'More Simple Sample Data');
}
/**
* @Route("/search/{query}", defaults={"query"=null}, name="elasticsearch.search"))
*/
public function searchAction($query, Request $request)
{
try {
$search = new Search();
//Save the form (uses POST method)
if ($request->getMethod() == "POST") {
// $request->query->get('search_form')['name'] = $request->request->get('form')['name'];
$request->request->set('search_form', $request->query->get('search_form'));
$form = $this->createForm(SearchFormType::class, $search);
$form->handleRequest($request);
/** @var Search $search */
$search = $form->getData();
$search->setName($request->request->get('form')['name']);
$search->setUser($this->getUser()->getUsername());
/** @var SearchFilter $filter */
foreach ($search->getFilters() as $filter) {
$filter->setSearch($search);
}
$em = $this->getDoctrine()->getManager();
$em->persist($search);
$em->flush();
return $this->redirectToRoute('elasticsearch.search', ['searchId' => $search->getId()]);
}
if (null != $request->query->get('page')) {
$page = $request->query->get('page');
} else {
$page = 1;
}
//Use search from a saved form
$searchId = $request->query->get('searchId');
if (null != $searchId) {
$em = $this->getDoctrine()->getManager();
$repository = $em->getRepository('AppBundle:Form\\Search');
$search = $repository->find($request->query->get('searchId'));
if (!$search) {
$this->createNotFoundException('Preset search not found');
}
}
$form = $this->createForm(SearchFormType::class, $search, ['method' => 'GET', 'savedSearch' => $searchId]);
$form->handleRequest($request);
$openSearchForm = $form->get('search')->isClicked();
//Form treatement after the "Save" button has been pressed (= ask for a name to save the search preset)
if ($form->isValid() && $request->query->get('search_form') && array_key_exists('save', $request->query->get('search_form'))) {
$form = $this->createFormBuilder($search)->add('name', \Symfony\Component\Form\Extension\Core\Type\TextType::class)->add('save_search', SubmitEmsType::class, ['label' => 'Save', 'attr' => ['class' => 'btn btn-primary pull-right'], 'icon' => 'fa fa-save'])->getForm();
return $this->render('elasticsearch/save-search.html.twig', ['form' => $form->createView()]);
} else {
if ($form->isValid() && $request->query->get('search_form') && array_key_exists('delete', $request->query->get('search_form'))) {
$this->addFlash('notice', 'Search has been deleted');
}
}
//Next we want 1. see the results, or 2. export the results
//So the common step is to fetch the results based on the search presets
/** @var Search $search */
if ($request->query->get('form') && array_key_exists('massExport', $request->query->get('form'))) {
//In case of export we saved the search object in json form, time to recuperate it
$jsonSearch = $request->query->get('form')['search-data'];
$encoders = array(new JsonEncoder());
$normalizers = array(new ObjectNormalizer());
$serializer = new Serializer($normalizers, $encoders);
$searchArray = json_decode($jsonSearch, true);
$filtersArray = $searchArray['filters'];
$searchArray['filters'] = null;
$search = $serializer->deserialize(json_encode($searchArray), Search::class, 'json');
foreach ($filtersArray as $rawFilter) {
$jsonFilter = json_encode($rawFilter);
$filter = $serializer->deserialize($jsonFilter, SearchFilter::class, 'json');
$search->addFilter($filter);
}
} else {
$search = $form->getData();
}
$body = $this->getSearchService()->generateSearchBody($search);
//
/** @var EntityManager $em */
$em = $this->getDoctrine()->getManager();
/** @var ContentTypeRepository $contentTypeRepository */
$contentTypeRepository = $em->getRepository('AppBundle:ContentType');
$types = $contentTypeRepository->findAllAsAssociativeArray();
/** @var EnvironmentRepository $environmentRepository */
$environmentRepository = $em->getRepository('AppBundle:Environment');
$environments = $environmentRepository->findAllAsAssociativeArray('alias');
/** @var \Elasticsearch\Client $client */
$client = $this->get('app.elasticsearch');
$assocAliases = $client->indices()->getAliases();
$mapAlias = [];
$mapIndex = [];
foreach ($assocAliases as $index => $aliasNames) {
foreach ($aliasNames['aliases'] as $alias => $options) {
if (isset($environments[$alias])) {
$mapAlias[$environments[$alias]['alias']] = $environments[$alias];
$mapIndex[$index] = $environments[$alias];
break;
}
}
}
$selectedEnvironments = [];
if (!empty($search->getEnvironments())) {
foreach ($search->getEnvironments() as $envName) {
$temp = $this->getEnvironmentService()->getAliasByName($envName);
if ($temp) {
$selectedEnvironments[] = $temp->getAlias();
}
}
}
//1. Define the parameters for a regular search request
$params = ['version' => true, 'index' => empty($selectedEnvironments) ? array_keys($environments) : $selectedEnvironments, 'type' => empty($search->getContentTypes()) ? array_keys($types) : array_values($search->getContentTypes()), 'size' => $this->container->getParameter('paging_size'), 'from' => ($page - 1) * $this->container->getParameter('paging_size')];
//2. Override parameters because when exporting we need all results, not paged
if ($request->query->get('form') && array_key_exists('massExport', $request->query->get('form'))) {
//TODO: size 10000 is the default maximum size of an elasticsearch installation. In case of export it would be better to use the scroll API of elasticsearch in case of performance issues. Or when more then 10000 results are going to be exported.
//TODO: consideration: will there be an export limit? Because for giant loads of data it might be better to call an API of the system that needs our exported data. Then again, they could simply connect to elasticsearch as a standalone application!
$params['from'] = 0;
$params['size'] = 10000;
}
// "highlight": {
// "fields": {
// "_all": {}
// }
// },
$body = array_merge($body, json_decode('{
"aggs": {
"types": {
"terms": {
"field": "_type",
"size": 15
}
},
"indexes": {
"terms": {
"field": "_index",
"size": 15
}
}
}
}', true));
$params['body'] = $body;
try {
$results = $client->search($params);
$lastPage = ceil($results['hits']['total'] / $this->container->getParameter('paging_size'));
} catch (ElasticsearchException $e) {
$this->addFlash('warning', $e->getMessage());
$lastPage = 0;
$results = ['hits' => ['total' => 0]];
}
$currentFilters = $request->query;
$currentFilters->remove('search_form[_token]');
//Form treatement after the "Export results" button has been pressed (= ask for a "content type" <-> "template" mapping)
if ($form->isValid() && $request->query->get('search_form') && array_key_exists('exportResults', $request->query->get('search_form'))) {
//Store all the content types present in the current resultset
$contentTypes = $this->getAllContentType($results);
/**@var ContentTypeService $contenttypeService*/
$contenttypeService = $this->get('ems.service.contenttype');
/**@var EnvironmentService $environmentService*/
$environmentService = $this->get('ems.service.environment');
//Check for each content type that an export template is available.
//If no export template is defined, ignore the content type.
//If one or more export templates are defined, allow choice of the template to be dynamic
$form = null;
foreach ($contentTypes as $name) {
/** @var ContentType $contentType */
$contentType = $types[$name];
$templateChoices = ['JSON export' => 0];
/** @var Template $template */
foreach ($contentType->getTemplates() as $template) {
if (RenderOptionType::EXPORT == $template->getRenderOption() && $template->getBody()) {
$templateChoices[$template->getName()] = $template->getId();
}
}
if (!empty($templateChoices)) {
if (!$form) {
$encoders = array(new JsonEncoder());
$normalizers = array(new ObjectNormalizer());
$serializer = new Serializer($normalizers, $encoders);
$jsonSearch = $serializer->serialize($search, 'json');
$form = $this->createFormBuilder()->setMethod('GET')->add('search-data', HiddenType::class, array('data' => $jsonSearch));
}
$form->add($name, ChoiceType::class, array('label' => 'Export template for ' . $contenttypeService->getByName($name)->getPluralName(), 'choices' => $templateChoices));
}
}
if ($form) {
$form = $form->add('massExport', SubmitType::class)->getForm();
$form->handlerequest($request);
return $this->render('elasticsearch/export-search.html.twig', ['form' => $form->createView()]);
} else {
return $this->render('elasticsearch/export-search.html.twig');
}
}
//Form treatement after the "Mass export" button has been pressed (= download all the results with the given preset)
if ($request->query->get('form') && array_key_exists('massExport', $request->query->get('form'))) {
//TODO: ? CANNOT DO THE ISVALID CHECK HERE :(
//Load the selected templates for each content type
/** @var EntityManager $em */
$em = $this->getDoctrine()->getManager();
/** @var ContentTypeRepository $repository */
$templateRepository = $em->getRepository('AppBundle:Template');
$templateChoises = $request->query->get('form');
$templateMapping = [];
$templateBodyMapping = [];
$twig = $this->getTwig();
$errorList = [];
foreach ($templateChoises as $contentName => $templateChoise) {
if ('search-data' != $contentName && 'massExport' != $contentName && '_token' != $contentName) {
$template = $templateRepository->find($templateChoise);
if ($template) {
$templateMapping[$contentName] = $template;
try {
//TODO why is the body generated and passed to the twig file while the twig file does not use it?
//Asked by dame
//If there is an error in the twig the user will get an 500 error page, this solution is not perfect but at least the template is tested
$body = $twig->createTemplate($template->getBody());
} catch (\Twig_Error $e) {
$this->addFlash('error', 'There is something wrong with the template ' . $template->getName());
$body = $twig->createTemplate('error in the template!');
$errorList[] = "Error in template->getBody() for: " . $template->getName();
continue;
}
$templateBodyMapping[$contentName] = $body;
} else {
//Default JSON export
$templateMapping[$contentName] = NULL;
$templateBodyMapping[$contentName] = NULL;
}
}
}
//Create the file for each result and accumulate in a zip stream
$extime = ini_get('max_execution_time');
ini_set('max_execution_time', 0);
$fileTime = date("D, d M Y H:i:s T");
$zip = new ZipStream("eMSExport.zip");
$contentTypes = $this->getAllContentType($results);
$resultsSize = count($results['hits']['hits']);
$loop = [];
$accumulatedContent = "";
foreach ($results['hits']['hits'] as $result) {
if (array_key_exists('first', $loop)) {
$loop['first'] = false;
} else {
$loop['first'] = true;
}
if (array_key_exists('index0', $loop)) {
$loop['index0'] = $loop['index0'] + 1;
} else {
$loop['index0'] = 0;
}
if (array_key_exists('index1', $loop)) {
$loop['index1'] = $loop['index1'] + 1;
} else {
$loop['index1'] = 1;
}
$loop['last'] = $resultsSize == $loop['index1'];
$name = $result['_type'];
$template = $templateMapping[$name];
$body = $templateBodyMapping[$name];
if ($template) {
$filename = $result['_id'];
if (null != $template->getFilename()) {
try {
$filename = $twig->createTemplate($template->getFilename());
} catch (\Twig_Error $e) {
$this->addFlash('error', 'There is something wrong with the template filename field ' . $template->getName());
$filename = $result['_id'];
$errorList[] = "Error in template->getFilename() for: " . $filename;
continue;
}
$filename = $filename->render(['loop' => $loop, 'contentType' => $template->getContentType(), 'object' => $result, 'source' => $result['_source']]);
$filename = preg_replace('~[\\r\\n]+~', '', $filename);
}
if (null != $template->getExtension()) {
$filename = $filename . '.' . $template->getExtension();
}
try {
$content = $body->render(['loop' => $loop, 'contentType' => $template->getContentType(), 'object' => $result, 'source' => $result['_source']]);
} catch (\Twig_Error $e) {
$this->addFlash('error', 'There is something wrong with the template filename field ' . $template->getName());
$content = "There was an error rendering the content";
$errorList[] = "Error in templateBody->render() for: " . $filename;
continue;
}
if ($template->getAccumulateInOneFile()) {
$accumulatedContent = $accumulatedContent . $content;
if ($loop['last']) {
$zip->addFile($template->getName() . '.' . $template->getExtension(), $accumulatedContent);
}
} else {
$zip->addFile($filename, $content);
}
} else {
//JSON export
$zip->addFile($result['_type'] . ' ' . $result['_id'] . '.json', json_encode($result['_source']));
}
}
if (!empty($errorList)) {
$zip->addFile("All-Errors.txt", implode("\n", $errorList));
}
$zip->finish();
exit;
}
return $this->render('elasticsearch/search.html.twig', ['results' => $results, 'lastPage' => $lastPage, 'paginationPath' => 'elasticsearch.search', 'types' => $types, 'alias' => $mapAlias, 'indexes' => $mapIndex, 'form' => $form->createView(), 'page' => $page, 'searchId' => $searchId, 'currentFilters' => $request->query, 'body' => $body, 'openSearchForm' => $openSearchForm, 'search' => $search]);
} catch (\Elasticsearch\Common\Exceptions\NoNodesAvailableException $e) {
return $this->redirectToRoute('elasticsearch.status');
}
}