public function etl($filename) { if (Kohana::$profiling) { $token = Profiler::start('Import', __FUNCTION__); } $buffers = array(); $buffers['data'] = array(); $buffers['log'] = array(); $files = Import_Helper::unzip($filename); // scanning workspace for uncompressed files foreach ($files as $file) { // check if it's a Mean-Archive $bname = basename($file); if (preg_match('/^Mean\\..*\\.xml\\.zip$/i', $bname)) { $type = 'data'; } elseif (preg_match('/^Log\\..*\\.xml\\.zip$/i', $bname) and $this->load_logs) { $type = 'log'; } else { continue; } $innerFiles = Import_Helper::unzip($file); foreach ($innerFiles as $xmlFile) { $xml = new SimpleXMLElement($xmlFile, NULL, true); $buffers[$type] = array_merge($buffers[$type], call_user_func(array($this, "_read_{$type}"), $xml)); unset($xml); unlink($xmlFile); } unlink($file); } $extracts = array(); foreach ($buffers as $type => $buffer) { $extracts += call_user_func(array($this, "_load_{$type}"), $buffer); } if (Kohana::$profiling) { Profiler::stop($token); } return $extracts; }
/** * Extracts the archive, * Transforms the channels and * Loads the data * * @param string $file * @return void * @throws Import_Exception */ public function import_file($file) { if (Kohana::$profiling) { $token = Profiler::start('Import', __FUNCTION__); } $data = false; try { Kohana::$log->add(Log::DEBUG, "Import start (:file)", array(':file' => $file)); $this->config = Kohana::$config->load('import'); // Get file path and check if the file exsits $file_path = Import_Helper::path($this->config->get('path')) . $file; if (!file_exists($file_path)) { throw new Import_Exception("File does not exist or is not readable. (:path)", array(':path' => $file_path)); } // Setup the workspace $temp_path = Import_Helper::path($this->config->get('workspace', sys_get_temp_dir())); if (file_exists($temp_path)) { $workspace = Import_Helper::path($temp_path . md5($file)); if (file_exists($workspace) or mkdir($workspace)) { chdir($workspace); // Create a copy to the workspace $working_copy = $workspace . $file; copy($file_path, $working_copy); Kohana::$log->add(Log::DEBUG, "Created temp directory ':path'", array(':path' => $workspace)); } else { throw new Import_Exception("Can't create temp directory ':path'", array(':path' => $workspace)); } } else { throw new Import_Exception("Workspace does not exist or is not readable. (:path)", array(':path' => $temp_path)); } $archive = $this->config->get('archive', FALSE); if ($archive) { $archive = Import_Helper::path($archive); if (file_exists($archive)) { $file_archive = $archive . $file; } else { throw new Import_Exception("Archive does not exist or is not readable. (:path)", array(':path' => $archive)); } } $bad_path = Import_Helper::path($this->config->get('bad_path')); if ($bad_path) { $file_bad = $bad_path . $file; } else { throw new Import_Exception("Badfile path does not exist or is not readable. (:path)", array(':path' => $bad_path)); } // setting up Schema object $schema_name = $this->config->get('schema', self::$defaults['schema']); $schema = Import_Schema::factory($schema_name); Kohana::$log->add(Log::DEBUG, "Using schema :schema", array(':schema' => $schema_name)); //////////////////////////////////////////////////////////////////// // ETL Data $schema->ch_filter = $this->config->get('ch_filter', self::$defaults['ch_filter']); $schema->ch_filter_type = $this->config->get('ch_filter_type', self::$defaults['ch_filter_type']); $schema->load_logs = $this->config->get('load_logs', self::$defaults['load_logs']); $schema->overwrite = $this->config->get('overwrite', self::$defaults['overwrite']); $data = $schema->etl($working_copy); $cnt = count($data); $this->data = array_merge($this->data, $data); // Cleaning up workspace Kohana::$log->add(Log::DEBUG, 'Cleanup temp directory.'); $objects = scandir($workspace); foreach ($objects as $object) { if ($object != "." && $object != "..") { unlink($workspace . "/" . $object); } } Kohana::$log->add(Log::DEBUG, 'Deleting temp directory.'); chdir(dirname($workspace)); rmdir($workspace); // Archive if ($archive and $data !== FALSE) { copy($file_path, $file_archive); } if (!$this->config->get('test_load', FALSE)) { unlink($file_path); } } catch (Kohana_Exception $e) { // copy file to the bad-Directory @copy($file_path, $file_bad); if (!isset($cnt)) { $cnt = 0; } Kohana::$log->add(Log::DEBUG, "Import aborted [:i rows affected] (:file)", array(':i' => $cnt, ':file' => $file)); if ($this->config instanceof Kohana_Config_Group and $this->config->get('throw_exceptions', self::$throw_exceptions)) { throw new Import_Exception("Cannot import file ':file'. :message", array(':file' => $file, ':message' => $e->getMessage())); } return; } Kohana::$log->add(Log::DEBUG, "Import end [:i rows affected] (:file)", array(':i' => $cnt, ':file' => $file)); if (Kohana::$profiling) { Profiler::stop($token); } }