/** * @depends testExportByModelIds */ public function testExportReportWithMultiplePagesOfData() { $super = User::getByUsername('super'); Yii::app()->user->userModel = $super; $numberOfUserNotifications = Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel); $report = new Report(); $report->setType(Report::TYPE_ROWS_AND_COLUMNS); $report->setModuleClassName('AccountsModule'); $report->setFiltersStructure(''); $displayAttribute = new DisplayAttributeForReportForm('AccountsModule', 'Account', Report::TYPE_ROWS_AND_COLUMNS); $displayAttribute->setModelAliasUsingTableAliasName('model1'); $displayAttribute->attributeIndexOrDerivedType = 'name'; $report->addDisplayAttribute($displayAttribute); $dataProvider = new RowsAndColumnsReportDataProvider($report); $exportItem = new ExportItem(); $exportItem->isCompleted = 0; $exportItem->exportFileType = 'csv'; $exportItem->exportFileName = 'rowAndColumnsTest2'; $exportItem->modelClassName = 'SavedReport'; $exportItem->serializedData = ExportUtil::getSerializedDataForExport($dataProvider); $this->assertTrue($exportItem->save()); $id = $exportItem->id; $exportItem->forget(); unset($exportItem); //Delete queued jobs from test exportItems created above Yii::app()->jobQueue->deleteAll(); ExportModule::$asynchronousPageSize = 2; $job = new ExportJob(); $this->assertEquals(0, count(Yii::app()->jobQueue->getAll())); $this->assertTrue($job->run()); $this->assertEquals(0, count(Yii::app()->jobQueue->getAll())); $exportItem = ExportItem::getById($id); $fileModel = $exportItem->exportFileModel; $this->assertEquals(1, $exportItem->isCompleted); $this->assertEquals(0, $exportItem->processOffset); $this->assertEquals('csv', $exportItem->exportFileType); $this->assertEquals('rowAndColumnsTest2', $exportItem->exportFileName); $this->assertTrue($fileModel instanceof ExportFileModel); $accounts = Account::getAll(); $headerData = array('Name'); $data = array(); foreach ($accounts as $account) { $data[] = array($account->name); } $output = ExportItemToCsvFileUtil::export($data, $headerData, 'rowAndColumnsTest2.csv', false); $this->assertEquals($output, $fileModel->fileContent->content); // Check if user got notification message, and if its type is ExportProcessCompleted $this->assertEquals($numberOfUserNotifications + 1, Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel)); //Matrix report should not paginate $report = new Report(); $report->setType(Report::TYPE_MATRIX); $report->setModuleClassName('AccountsModule'); $report->setFiltersStructure(''); $displayAttribute = new DisplayAttributeForReportForm('AccountsModule', 'Account', Report::TYPE_MATRIX); $displayAttribute->setModelAliasUsingTableAliasName('model1'); $displayAttribute->attributeIndexOrDerivedType = 'Count'; $report->addDisplayAttribute($displayAttribute); $groupBy = new GroupByForReportForm('AccountsModule', 'Account', Report::TYPE_MATRIX); $groupBy->attributeIndexOrDerivedType = 'name'; $groupBy->axis = 'y'; $report->addGroupBy($groupBy); $groupBy = new GroupByForReportForm('AccountsModule', 'Account', Report::TYPE_MATRIX); $groupBy->attributeIndexOrDerivedType = 'officePhone'; $report->addGroupBy($groupBy); $dataProvider = new MatrixReportDataProvider($report); $exportItem = new ExportItem(); $exportItem->isCompleted = 0; $exportItem->exportFileType = 'csv'; $exportItem->exportFileName = 'matrixTest1'; $exportItem->modelClassName = 'SavedReport'; $exportItem->serializedData = ExportUtil::getSerializedDataForExport($dataProvider); $this->assertTrue($exportItem->save()); $id = $exportItem->id; $exportItem->forget(); unset($exportItem); //Delete queued jobs from test exportItems created above Yii::app()->jobQueue->deleteAll(); ExportModule::$asynchronousPageSize = 2; $job = new ExportJob(); $this->assertTrue($job->run()); $exportItem = ExportItem::getById($id); $fileModel = $exportItem->exportFileModel; $this->assertEquals(1, $exportItem->isCompleted); $this->assertEquals(0, $exportItem->processOffset); $this->assertEquals('csv', $exportItem->exportFileType); $this->assertEquals('matrixTest1', $exportItem->exportFileName); $this->assertTrue($fileModel instanceof ExportFileModel); $fileContent = $fileModel->fileContent->content; $this->assertContains('Test Account', $fileContent); $this->assertContains('Test Account 2', $fileContent); $this->assertContains('Test Account 3', $fileContent); $this->assertContains('Test Account 4', $fileContent); // Check if user got notification message, and if its type is ExportProcessCompleted $this->assertEquals($numberOfUserNotifications + 2, Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel)); }
/** * Test if background export job generated csv file, * check if content of this csv file is correct, and * finally check if user got notification message that * his downloads are completed. */ public function testRun() { $super = User::getByUsername('super'); Yii::app()->user->userModel = $super; $numberOfUserNotifications = Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel); $account = new Account(); $account->owner = $super; $account->name = 'Test Account'; $account->officePhone = '1234567890'; $this->assertTrue($account->save()); $account = new Account(); $account->owner = $super; $account->name = 'Test Account 2'; $account->officePhone = '1234567899'; $this->assertTrue($account->save()); $account = new Account(false); $searchForm = new AccountsSearchForm($account); $dataProvider = ExportTestHelper::makeRedBeanDataProvider($searchForm, 'Account', 0, Yii::app()->user->userModel->id); $totalItems = $dataProvider->getTotalItemCount(); $this->assertEquals(2, $totalItems); $exportItem = new ExportItem(); $exportItem->isCompleted = 0; $exportItem->exportFileType = 'csv'; $exportItem->exportFileName = 'test'; $exportItem->modelClassName = 'Account'; $exportItem->serializedData = serialize($dataProvider); $this->assertTrue($exportItem->save()); $id = $exportItem->id; $exportItem->forget(); unset($exportItem); $job = new ExportJob(); $this->assertTrue($job->run()); $exportItem = ExportItem::getById($id); $fileModel = $exportItem->exportFileModel; $this->assertEquals(1, $exportItem->isCompleted); $this->assertEquals('csv', $exportItem->exportFileType); $this->assertEquals('test', $exportItem->exportFileName); $this->assertTrue($fileModel instanceof ExportFileModel); // Get csv string via regular csv export process(directly, not in background) // We suppose that csv generated thisway is corrected, this function itself // is tested in another test. $data = array(); $rows = $dataProvider->getData(); $modelToExportAdapter = new ModelToExportAdapter($rows[0]); $headerData = $modelToExportAdapter->getHeaderData(); foreach ($rows as $model) { $modelToExportAdapter = new ModelToExportAdapter($model); $data[] = $modelToExportAdapter->getData(); } $output = ExportItemToCsvFileUtil::export($data, $headerData, 'test.csv', false); $this->assertEquals($output, $fileModel->fileContent->content); // Check if user got notification message, and if its type is ExportProcessCompleted $this->assertEquals($numberOfUserNotifications + 1, Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel)); }
public function actionExport($id, $stickySearchKey = null) { assert('$stickySearchKey == null || is_string($stickySearchKey)'); $savedReport = SavedReport::getById((int) $id); ControllerSecurityUtil::resolveCanCurrentUserAccessModule($savedReport->moduleClassName); ControllerSecurityUtil::resolveAccessCanCurrentUserReadModel($savedReport); $report = SavedReportToReportAdapter::makeReportBySavedReport($savedReport); $dataProvider = $this->getDataProviderForExport($report, $report->getId(), false); $totalItems = intval($dataProvider->calculateTotalItemCount()); $data = array(); if ($totalItems > 0) { if ($totalItems <= ExportModule::$asynchronousThreshold) { // Output csv file directly to user browser if ($dataProvider) { $reportToExportAdapter = ReportToExportAdapterFactory::createReportToExportAdapter($report, $dataProvider); $headerData = $reportToExportAdapter->getHeaderData(); $data = $reportToExportAdapter->getData(); } // Output data if (count($data)) { $fileName = $this->getModule()->getName() . ".csv"; ExportItemToCsvFileUtil::export($data, $headerData, $fileName, true); } else { Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'There is no data to export.')); } } else { if ($dataProvider) { $serializedData = ExportUtil::getSerializedDataForExport($dataProvider); } // Create background job $exportItem = new ExportItem(); $exportItem->isCompleted = 0; $exportItem->exportFileType = 'csv'; $exportItem->exportFileName = $this->getModule()->getName(); $exportItem->modelClassName = 'SavedReport'; $exportItem->serializedData = $serializedData; $exportItem->save(); $exportItem->forget(); Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'A large amount of data has been requested for export. You will receive ' . 'a notification with the download link when the export is complete.')); } } else { Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'There is no data to export.')); } $this->redirect(array($this->getId() . '/index')); }
protected static function generateCsvAndReturnFileName($sizeInMB) { echo "Setting up dummy csv file of {$sizeInMB}MB" . PHP_EOL; $startTime = microtime(true); if (!isset(static::$temporaryFilePath)) { static::$temporaryFilePath = tempnam(sys_get_temp_dir(), "benchmark_csv_"); } if (file_exists(static::$temporaryFilePath)) { $existingSize = floor(filesize(static::$temporaryFilePath) / 1048576); $sizeInMB -= $existingSize; } $itemCount = $sizeInMB * 1018; $totalLength = 1024; // how long each each row should be in total. $data = array(); for ($i = 0; $i < $itemCount; $i++) { $remainingLength = $totalLength; do { $lengthOfCurrentSegment = rand(1, $remainingLength); $data[$i][] = str_repeat('_', $lengthOfCurrentSegment); $remainingLength -= $lengthOfCurrentSegment; } while ($remainingLength > 0); } $csv = ExportItemToCsvFileUtil::export($data); // this doesn't matter even if its not binary, no special characters in this string file_put_contents(static::$temporaryFilePath, $csv, FILE_APPEND); $endTime = microtime(true); $difference = number_format($endTime - $startTime, 3); echo "CSV generated in: {$difference} seconds." . PHP_EOL; }
protected function export($stickySearchKey = null, $modelClassName = null, $exportFileName = null) { assert('$stickySearchKey == null || is_string($stickySearchKey)'); assert('$modelClassName == null || is_string($modelClassName)'); assert('$exportFileName == null || is_string($exportFileName)'); if ($modelClassName == null) { $modelClassName = $this->getModelName(); } $searchFormClassName = static::getSearchFormClassName(); $pageSize = null; $model = new $modelClassName(false); if ($searchFormClassName != null) { $searchForm = new $searchFormClassName($model); } else { throw new NotSupportedException(); } $stateMetadataAdapterClassName = $this->resolveStateMetadataAdapterClassNameForExport(); $dataProvider = $this->getDataProviderByResolvingSelectAllFromGet($searchForm, $pageSize, Yii::app()->user->userModel->id, $stateMetadataAdapterClassName, $stickySearchKey); if (!$dataProvider) { $idsToExport = array_filter(explode(",", trim($_GET['selectedIds'], " ,"))); // Not Coding Standard } $totalItems = static::getSelectedRecordCountByResolvingSelectAllFromGet($dataProvider, false); $data = array(); if ($totalItems > 0) { if ($totalItems <= ExportModule::$asynchronousThreshold) { // Output csv file directly to user browser if ($dataProvider) { $modelsToExport = ExportUtil::getDataForExport($dataProvider); if (count($modelsToExport) > 0) { $modelToExportAdapter = new ModelToExportAdapter($modelsToExport[0]); $headerData = $modelToExportAdapter->getHeaderData(); } foreach ($modelsToExport as $model) { if (ControllerSecurityUtil::doesCurrentUserHavePermissionOnSecurableItem($model, Permission::READ)) { $modelToExportAdapter = new ModelToExportAdapter($model); $data[] = $modelToExportAdapter->getData(); } } } else { $headerData = array(); foreach ($idsToExport as $idToExport) { $model = $modelClassName::getById(intval($idToExport)); if (ControllerSecurityUtil::doesCurrentUserHavePermissionOnSecurableItem($model, Permission::READ)) { $modelToExportAdapter = new ModelToExportAdapter($model); $data[] = $modelToExportAdapter->getData(); if (count($headerData) == 0) { $headerData = $modelToExportAdapter->getHeaderData(); } } } } // Output data if (count($data)) { if ($exportFileName == null) { $fileName = $this->getModule()->getName() . ".csv"; } else { $fileName = $exportFileName . ".csv"; } $output = ExportItemToCsvFileUtil::export($data, $headerData, $fileName, true); } else { Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'There is no data to export.')); } } else { if ($dataProvider) { $serializedData = ExportUtil::getSerializedDataForExport($dataProvider); } else { $serializedData = serialize($idsToExport); } // Create background job $exportItem = new ExportItem(); $exportItem->isCompleted = false; $exportItem->exportFileType = 'csv'; $exportItem->exportFileName = $this->getModule()->getName(); $exportItem->modelClassName = $modelClassName; $exportItem->serializedData = $serializedData; $exportItem->isJobRunning = false; $exportItem->cancelExport = false; $exportItem->save(); $exportItem->forget(); Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'A large amount of data has been requested for export. You will receive ' . 'a notification with the download link when the export is complete.')); } } else { Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'There is no data to export.')); } $this->redirect(array($this->getId() . '/index')); }
protected function isValidCsvConversion($textAreaContent) { $super = User::getByUsername('super'); Yii::app()->user->userModel = $super; $testItem = new ExportTestModelItem(); $testItem->firstName = 'Bob3'; $testItem->lastName = 'Bob3'; $testItem->boolean = true; $testItem->date = '2002-04-03'; $testItem->dateTime = '2002-04-03 02:00:43'; $testItem->float = 54.22; $testItem->integer = 10; $testItem->phone = '21313213'; $testItem->string = 'aString'; $testItem->textArea = $textAreaContent; $testItem->url = 'http://www.asite.com'; $testItem->email = '*****@*****.**'; $testItem->save(); $id = $testItem->id; $testItem->forget(); unset($testItem); $data = array(); $testItem = ExportTestModelItem::getById($id); $adapter = new ModelToExportAdapter($testItem); $data[] = $adapter->getData(); $headerData = $adapter->getHeaderData(); // Export data to csv, and then revert csv back to array, so we compare data $csvData = ExportItemToCsvFileUtil::export($data, $headerData, 'exports.csv', false); $revertedData = CsvParser::parseFromString($csvData); // We are testing ModelToExportAdapter in details in another test // so in this test we suppose that ModelToExportAdapter::getData // return correct results $adapter = new ModelToExportAdapter($testItem); $compareData = $adapter->getData(); $compareHeaderData = $adapter->getHeaderData(); // Using === here would fail as we are not setting all keys part of getData()'s return array if ($compareData == array_values($revertedData[0]) && $compareHeaderData == array_keys($revertedData[0])) { return true; } return false; }
/** * Converts import array to csv * @param $importArray * @param $firstRowIsHeaderRow * @return string */ protected static function convertImportArrayToCsv(array $importArray, $firstRowIsHeaderRow) { $headerArray = array(); if ($firstRowIsHeaderRow) { $headerArray = array_shift($importArray); } $csv = ExportItemToCsvFileUtil::export($importArray, $headerArray, '', false, true, true); return $csv; }
/** * @param ExportItem $exportItem * @param $idsToExport */ protected function processIdsToExport(ExportItem $exportItem, $idsToExport) { $headerData = array(); $data = array(); $models = array(); foreach ($idsToExport as $idToExport) { $models[] = call_user_func(array($exportItem->modelClassName, 'getById'), intval($idToExport)); $this->totalModelsProcessed++; } $this->processExportModels($models, $headerData, $data); $content = ExportItemToCsvFileUtil::export($data, $headerData); $exportFileModel = $this->makeExportFileModelByContent($content, $exportItem->exportFileName); $this->processCompletedExportItem($exportItem, $exportFileModel); }
public function run() { $exportItems = ExportItem::getUncompletedItems(); if (count($exportItems) > 0) { foreach ($exportItems as $exportItem) { if (isset($exportItem->exportFileModel)) { //continue; } $unserializedData = unserialize($exportItem->serializedData); if ($unserializedData instanceof RedBeanModelDataProvider) { $formattedData = $unserializedData->getData(); } else { $formattedData = array(); foreach ($unserializedData as $idToExport) { $model = call_user_func(array($exportItem->modelClassName, 'getById'), intval($idToExport)); $formattedData[] = $model; } } if ($exportItem->exportFileType == 'csv') { $headerData = array(); foreach ($formattedData as $model) { if (ControllerSecurityUtil::doesCurrentUserHavePermissionOnSecurableItem($model, Permission::READ)) { $modelToExportAdapter = new ModelToExportAdapter($model); if (count($headerData) == 0) { $headerData = $modelToExportAdapter->getHeaderData(); } $data[] = $modelToExportAdapter->getData(); } } $output = ExportItemToCsvFileUtil::export($data, $headerData); $fileContent = new FileContent(); $fileContent->content = $output; $exportFileModel = new ExportFileModel(); $exportFileModel->fileContent = $fileContent; $exportFileModel->name = $exportItem->exportFileName . ".csv"; $exportFileModel->type = 'application/octet-stream'; $exportFileModel->size = strlen($output); $saved = $exportFileModel->save(); if ($saved) { $exportItem->isCompleted = 1; $exportItem->exportFileModel = $exportFileModel; $exportItem->save(); $message = new NotificationMessage(); $message->htmlContent = Zurmo::t('ExportModule', 'Export of {fileName} requested on {dateTime} is completed. <a href="{url}">Click here</a> to download file!', array('{fileName}' => $exportItem->exportFileName, '{url}' => Yii::app()->createUrl('export/default/download', array('id' => $exportItem->id)), '{dateTime}' => DateTimeUtil::convertDbFormattedDateTimeToLocaleFormattedDisplay($exportItem->createdDateTime, 'long'))); $rules = new ExportProcessCompletedNotificationRules(); NotificationsUtil::submit($message, $rules); } } } } else { return true; } return true; }