/**
* @depends testExportByModelIds
*/
public function testExportReportWithMultiplePagesOfData()
{
$super = User::getByUsername('super');
Yii::app()->user->userModel = $super;
$numberOfUserNotifications = Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel);
$report = new Report();
$report->setType(Report::TYPE_ROWS_AND_COLUMNS);
$report->setModuleClassName('AccountsModule');
$report->setFiltersStructure('');
$displayAttribute = new DisplayAttributeForReportForm('AccountsModule', 'Account', Report::TYPE_ROWS_AND_COLUMNS);
$displayAttribute->setModelAliasUsingTableAliasName('model1');
$displayAttribute->attributeIndexOrDerivedType = 'name';
$report->addDisplayAttribute($displayAttribute);
$dataProvider = new RowsAndColumnsReportDataProvider($report);
$exportItem = new ExportItem();
$exportItem->isCompleted = 0;
$exportItem->exportFileType = 'csv';
$exportItem->exportFileName = 'rowAndColumnsTest2';
$exportItem->modelClassName = 'SavedReport';
$exportItem->serializedData = ExportUtil::getSerializedDataForExport($dataProvider);
$this->assertTrue($exportItem->save());
$id = $exportItem->id;
$exportItem->forget();
unset($exportItem);
//Delete queued jobs from test exportItems created above
Yii::app()->jobQueue->deleteAll();
ExportModule::$asynchronousPageSize = 2;
$job = new ExportJob();
$this->assertEquals(0, count(Yii::app()->jobQueue->getAll()));
$this->assertTrue($job->run());
$this->assertEquals(0, count(Yii::app()->jobQueue->getAll()));
$exportItem = ExportItem::getById($id);
$fileModel = $exportItem->exportFileModel;
$this->assertEquals(1, $exportItem->isCompleted);
$this->assertEquals(0, $exportItem->processOffset);
$this->assertEquals('csv', $exportItem->exportFileType);
$this->assertEquals('rowAndColumnsTest2', $exportItem->exportFileName);
$this->assertTrue($fileModel instanceof ExportFileModel);
$accounts = Account::getAll();
$headerData = array('Name');
$data = array();
foreach ($accounts as $account) {
$data[] = array($account->name);
}
$output = ExportItemToCsvFileUtil::export($data, $headerData, 'rowAndColumnsTest2.csv', false);
$this->assertEquals($output, $fileModel->fileContent->content);
// Check if user got notification message, and if its type is ExportProcessCompleted
$this->assertEquals($numberOfUserNotifications + 1, Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel));
//Matrix report should not paginate
$report = new Report();
$report->setType(Report::TYPE_MATRIX);
$report->setModuleClassName('AccountsModule');
$report->setFiltersStructure('');
$displayAttribute = new DisplayAttributeForReportForm('AccountsModule', 'Account', Report::TYPE_MATRIX);
$displayAttribute->setModelAliasUsingTableAliasName('model1');
$displayAttribute->attributeIndexOrDerivedType = 'Count';
$report->addDisplayAttribute($displayAttribute);
$groupBy = new GroupByForReportForm('AccountsModule', 'Account', Report::TYPE_MATRIX);
$groupBy->attributeIndexOrDerivedType = 'name';
$groupBy->axis = 'y';
$report->addGroupBy($groupBy);
$groupBy = new GroupByForReportForm('AccountsModule', 'Account', Report::TYPE_MATRIX);
$groupBy->attributeIndexOrDerivedType = 'officePhone';
$report->addGroupBy($groupBy);
$dataProvider = new MatrixReportDataProvider($report);
$exportItem = new ExportItem();
$exportItem->isCompleted = 0;
$exportItem->exportFileType = 'csv';
$exportItem->exportFileName = 'matrixTest1';
$exportItem->modelClassName = 'SavedReport';
$exportItem->serializedData = ExportUtil::getSerializedDataForExport($dataProvider);
$this->assertTrue($exportItem->save());
$id = $exportItem->id;
$exportItem->forget();
unset($exportItem);
//Delete queued jobs from test exportItems created above
Yii::app()->jobQueue->deleteAll();
ExportModule::$asynchronousPageSize = 2;
$job = new ExportJob();
$this->assertTrue($job->run());
$exportItem = ExportItem::getById($id);
$fileModel = $exportItem->exportFileModel;
$this->assertEquals(1, $exportItem->isCompleted);
$this->assertEquals(0, $exportItem->processOffset);
$this->assertEquals('csv', $exportItem->exportFileType);
$this->assertEquals('matrixTest1', $exportItem->exportFileName);
$this->assertTrue($fileModel instanceof ExportFileModel);
$fileContent = $fileModel->fileContent->content;
$this->assertContains('Test Account', $fileContent);
$this->assertContains('Test Account 2', $fileContent);
$this->assertContains('Test Account 3', $fileContent);
$this->assertContains('Test Account 4', $fileContent);
// Check if user got notification message, and if its type is ExportProcessCompleted
$this->assertEquals($numberOfUserNotifications + 2, Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel));
}
/**
* Test if background export job generated csv file,
* check if content of this csv file is correct, and
* finally check if user got notification message that
* his downloads are completed.
*/
public function testRun()
{
$super = User::getByUsername('super');
Yii::app()->user->userModel = $super;
$numberOfUserNotifications = Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel);
$account = new Account();
$account->owner = $super;
$account->name = 'Test Account';
$account->officePhone = '1234567890';
$this->assertTrue($account->save());
$account = new Account();
$account->owner = $super;
$account->name = 'Test Account 2';
$account->officePhone = '1234567899';
$this->assertTrue($account->save());
$account = new Account(false);
$searchForm = new AccountsSearchForm($account);
$dataProvider = ExportTestHelper::makeRedBeanDataProvider($searchForm, 'Account', 0, Yii::app()->user->userModel->id);
$totalItems = $dataProvider->getTotalItemCount();
$this->assertEquals(2, $totalItems);
$exportItem = new ExportItem();
$exportItem->isCompleted = 0;
$exportItem->exportFileType = 'csv';
$exportItem->exportFileName = 'test';
$exportItem->modelClassName = 'Account';
$exportItem->serializedData = serialize($dataProvider);
$this->assertTrue($exportItem->save());
$id = $exportItem->id;
$exportItem->forget();
unset($exportItem);
$job = new ExportJob();
$this->assertTrue($job->run());
$exportItem = ExportItem::getById($id);
$fileModel = $exportItem->exportFileModel;
$this->assertEquals(1, $exportItem->isCompleted);
$this->assertEquals('csv', $exportItem->exportFileType);
$this->assertEquals('test', $exportItem->exportFileName);
$this->assertTrue($fileModel instanceof ExportFileModel);
// Get csv string via regular csv export process(directly, not in background)
// We suppose that csv generated thisway is corrected, this function itself
// is tested in another test.
$data = array();
$rows = $dataProvider->getData();
$modelToExportAdapter = new ModelToExportAdapter($rows[0]);
$headerData = $modelToExportAdapter->getHeaderData();
foreach ($rows as $model) {
$modelToExportAdapter = new ModelToExportAdapter($model);
$data[] = $modelToExportAdapter->getData();
}
$output = ExportItemToCsvFileUtil::export($data, $headerData, 'test.csv', false);
$this->assertEquals($output, $fileModel->fileContent->content);
// Check if user got notification message, and if its type is ExportProcessCompleted
$this->assertEquals($numberOfUserNotifications + 1, Notification::getCountByTypeAndUser('ExportProcessCompleted', Yii::app()->user->userModel));
}
public function actionExport($id, $stickySearchKey = null)
{
assert('$stickySearchKey == null || is_string($stickySearchKey)');
$savedReport = SavedReport::getById((int) $id);
ControllerSecurityUtil::resolveCanCurrentUserAccessModule($savedReport->moduleClassName);
ControllerSecurityUtil::resolveAccessCanCurrentUserReadModel($savedReport);
$report = SavedReportToReportAdapter::makeReportBySavedReport($savedReport);
$dataProvider = $this->getDataProviderForExport($report, $report->getId(), false);
$totalItems = intval($dataProvider->calculateTotalItemCount());
$data = array();
if ($totalItems > 0) {
if ($totalItems <= ExportModule::$asynchronousThreshold) {
// Output csv file directly to user browser
if ($dataProvider) {
$reportToExportAdapter = ReportToExportAdapterFactory::createReportToExportAdapter($report, $dataProvider);
$headerData = $reportToExportAdapter->getHeaderData();
$data = $reportToExportAdapter->getData();
}
// Output data
if (count($data)) {
$fileName = $this->getModule()->getName() . ".csv";
ExportItemToCsvFileUtil::export($data, $headerData, $fileName, true);
} else {
Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'There is no data to export.'));
}
} else {
if ($dataProvider) {
$serializedData = ExportUtil::getSerializedDataForExport($dataProvider);
}
// Create background job
$exportItem = new ExportItem();
$exportItem->isCompleted = 0;
$exportItem->exportFileType = 'csv';
$exportItem->exportFileName = $this->getModule()->getName();
$exportItem->modelClassName = 'SavedReport';
$exportItem->serializedData = $serializedData;
$exportItem->save();
$exportItem->forget();
Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'A large amount of data has been requested for export. You will receive ' . 'a notification with the download link when the export is complete.'));
}
} else {
Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'There is no data to export.'));
}
$this->redirect(array($this->getId() . '/index'));
}
protected static function generateCsvAndReturnFileName($sizeInMB)
{
echo "Setting up dummy csv file of {$sizeInMB}MB" . PHP_EOL;
$startTime = microtime(true);
if (!isset(static::$temporaryFilePath)) {
static::$temporaryFilePath = tempnam(sys_get_temp_dir(), "benchmark_csv_");
}
if (file_exists(static::$temporaryFilePath)) {
$existingSize = floor(filesize(static::$temporaryFilePath) / 1048576);
$sizeInMB -= $existingSize;
}
$itemCount = $sizeInMB * 1018;
$totalLength = 1024;
// how long each each row should be in total.
$data = array();
for ($i = 0; $i < $itemCount; $i++) {
$remainingLength = $totalLength;
do {
$lengthOfCurrentSegment = rand(1, $remainingLength);
$data[$i][] = str_repeat('_', $lengthOfCurrentSegment);
$remainingLength -= $lengthOfCurrentSegment;
} while ($remainingLength > 0);
}
$csv = ExportItemToCsvFileUtil::export($data);
// this doesn't matter even if its not binary, no special characters in this string
file_put_contents(static::$temporaryFilePath, $csv, FILE_APPEND);
$endTime = microtime(true);
$difference = number_format($endTime - $startTime, 3);
echo "CSV generated in: {$difference} seconds." . PHP_EOL;
}
protected function export($stickySearchKey = null, $modelClassName = null, $exportFileName = null)
{
assert('$stickySearchKey == null || is_string($stickySearchKey)');
assert('$modelClassName == null || is_string($modelClassName)');
assert('$exportFileName == null || is_string($exportFileName)');
if ($modelClassName == null) {
$modelClassName = $this->getModelName();
}
$searchFormClassName = static::getSearchFormClassName();
$pageSize = null;
$model = new $modelClassName(false);
if ($searchFormClassName != null) {
$searchForm = new $searchFormClassName($model);
} else {
throw new NotSupportedException();
}
$stateMetadataAdapterClassName = $this->resolveStateMetadataAdapterClassNameForExport();
$dataProvider = $this->getDataProviderByResolvingSelectAllFromGet($searchForm, $pageSize, Yii::app()->user->userModel->id, $stateMetadataAdapterClassName, $stickySearchKey);
if (!$dataProvider) {
$idsToExport = array_filter(explode(",", trim($_GET['selectedIds'], " ,")));
// Not Coding Standard
}
$totalItems = static::getSelectedRecordCountByResolvingSelectAllFromGet($dataProvider, false);
$data = array();
if ($totalItems > 0) {
if ($totalItems <= ExportModule::$asynchronousThreshold) {
// Output csv file directly to user browser
if ($dataProvider) {
$modelsToExport = ExportUtil::getDataForExport($dataProvider);
if (count($modelsToExport) > 0) {
$modelToExportAdapter = new ModelToExportAdapter($modelsToExport[0]);
$headerData = $modelToExportAdapter->getHeaderData();
}
foreach ($modelsToExport as $model) {
if (ControllerSecurityUtil::doesCurrentUserHavePermissionOnSecurableItem($model, Permission::READ)) {
$modelToExportAdapter = new ModelToExportAdapter($model);
$data[] = $modelToExportAdapter->getData();
}
}
} else {
$headerData = array();
foreach ($idsToExport as $idToExport) {
$model = $modelClassName::getById(intval($idToExport));
if (ControllerSecurityUtil::doesCurrentUserHavePermissionOnSecurableItem($model, Permission::READ)) {
$modelToExportAdapter = new ModelToExportAdapter($model);
$data[] = $modelToExportAdapter->getData();
if (count($headerData) == 0) {
$headerData = $modelToExportAdapter->getHeaderData();
}
}
}
}
// Output data
if (count($data)) {
if ($exportFileName == null) {
$fileName = $this->getModule()->getName() . ".csv";
} else {
$fileName = $exportFileName . ".csv";
}
$output = ExportItemToCsvFileUtil::export($data, $headerData, $fileName, true);
} else {
Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'There is no data to export.'));
}
} else {
if ($dataProvider) {
$serializedData = ExportUtil::getSerializedDataForExport($dataProvider);
} else {
$serializedData = serialize($idsToExport);
}
// Create background job
$exportItem = new ExportItem();
$exportItem->isCompleted = false;
$exportItem->exportFileType = 'csv';
$exportItem->exportFileName = $this->getModule()->getName();
$exportItem->modelClassName = $modelClassName;
$exportItem->serializedData = $serializedData;
$exportItem->isJobRunning = false;
$exportItem->cancelExport = false;
$exportItem->save();
$exportItem->forget();
Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'A large amount of data has been requested for export. You will receive ' . 'a notification with the download link when the export is complete.'));
}
} else {
Yii::app()->user->setFlash('notification', Zurmo::t('ZurmoModule', 'There is no data to export.'));
}
$this->redirect(array($this->getId() . '/index'));
}
protected function isValidCsvConversion($textAreaContent)
{
$super = User::getByUsername('super');
Yii::app()->user->userModel = $super;
$testItem = new ExportTestModelItem();
$testItem->firstName = 'Bob3';
$testItem->lastName = 'Bob3';
$testItem->boolean = true;
$testItem->date = '2002-04-03';
$testItem->dateTime = '2002-04-03 02:00:43';
$testItem->float = 54.22;
$testItem->integer = 10;
$testItem->phone = '21313213';
$testItem->string = 'aString';
$testItem->textArea = $textAreaContent;
$testItem->url = 'http://www.asite.com';
$testItem->email = '*****@*****.**';
$testItem->save();
$id = $testItem->id;
$testItem->forget();
unset($testItem);
$data = array();
$testItem = ExportTestModelItem::getById($id);
$adapter = new ModelToExportAdapter($testItem);
$data[] = $adapter->getData();
$headerData = $adapter->getHeaderData();
// Export data to csv, and then revert csv back to array, so we compare data
$csvData = ExportItemToCsvFileUtil::export($data, $headerData, 'exports.csv', false);
$revertedData = CsvParser::parseFromString($csvData);
// We are testing ModelToExportAdapter in details in another test
// so in this test we suppose that ModelToExportAdapter::getData
// return correct results
$adapter = new ModelToExportAdapter($testItem);
$compareData = $adapter->getData();
$compareHeaderData = $adapter->getHeaderData();
// Using === here would fail as we are not setting all keys part of getData()'s return array
if ($compareData == array_values($revertedData[0]) && $compareHeaderData == array_keys($revertedData[0])) {
return true;
}
return false;
}
/**
* Converts import array to csv
* @param $importArray
* @param $firstRowIsHeaderRow
* @return string
*/
protected static function convertImportArrayToCsv(array $importArray, $firstRowIsHeaderRow)
{
$headerArray = array();
if ($firstRowIsHeaderRow) {
$headerArray = array_shift($importArray);
}
$csv = ExportItemToCsvFileUtil::export($importArray, $headerArray, '', false, true, true);
return $csv;
}
/**
* @param ExportItem $exportItem
* @param $idsToExport
*/
protected function processIdsToExport(ExportItem $exportItem, $idsToExport)
{
$headerData = array();
$data = array();
$models = array();
foreach ($idsToExport as $idToExport) {
$models[] = call_user_func(array($exportItem->modelClassName, 'getById'), intval($idToExport));
$this->totalModelsProcessed++;
}
$this->processExportModels($models, $headerData, $data);
$content = ExportItemToCsvFileUtil::export($data, $headerData);
$exportFileModel = $this->makeExportFileModelByContent($content, $exportItem->exportFileName);
$this->processCompletedExportItem($exportItem, $exportFileModel);
}
public function run()
{
$exportItems = ExportItem::getUncompletedItems();
if (count($exportItems) > 0) {
foreach ($exportItems as $exportItem) {
if (isset($exportItem->exportFileModel)) {
//continue;
}
$unserializedData = unserialize($exportItem->serializedData);
if ($unserializedData instanceof RedBeanModelDataProvider) {
$formattedData = $unserializedData->getData();
} else {
$formattedData = array();
foreach ($unserializedData as $idToExport) {
$model = call_user_func(array($exportItem->modelClassName, 'getById'), intval($idToExport));
$formattedData[] = $model;
}
}
if ($exportItem->exportFileType == 'csv') {
$headerData = array();
foreach ($formattedData as $model) {
if (ControllerSecurityUtil::doesCurrentUserHavePermissionOnSecurableItem($model, Permission::READ)) {
$modelToExportAdapter = new ModelToExportAdapter($model);
if (count($headerData) == 0) {
$headerData = $modelToExportAdapter->getHeaderData();
}
$data[] = $modelToExportAdapter->getData();
}
}
$output = ExportItemToCsvFileUtil::export($data, $headerData);
$fileContent = new FileContent();
$fileContent->content = $output;
$exportFileModel = new ExportFileModel();
$exportFileModel->fileContent = $fileContent;
$exportFileModel->name = $exportItem->exportFileName . ".csv";
$exportFileModel->type = 'application/octet-stream';
$exportFileModel->size = strlen($output);
$saved = $exportFileModel->save();
if ($saved) {
$exportItem->isCompleted = 1;
$exportItem->exportFileModel = $exportFileModel;
$exportItem->save();
$message = new NotificationMessage();
$message->htmlContent = Zurmo::t('ExportModule', 'Export of {fileName} requested on {dateTime} is completed. <a href="{url}">Click here</a> to download file!', array('{fileName}' => $exportItem->exportFileName, '{url}' => Yii::app()->createUrl('export/default/download', array('id' => $exportItem->id)), '{dateTime}' => DateTimeUtil::convertDbFormattedDateTimeToLocaleFormattedDisplay($exportItem->createdDateTime, 'long')));
$rules = new ExportProcessCompletedNotificationRules();
NotificationsUtil::submit($message, $rules);
}
}
}
} else {
return true;
}
return true;
}