/**
* Process view
*
* @copyright
* @author RolandD
* @todo
* @see
* @access public
* @param
* @return
* @since 4.0
*/
public function display($tpl = null)
{
$jinput = JFactory::getApplication()->input;
// Load the settings
$this->loadHelper('settings');
$settings = new CsviSettings();
if ($settings->get('log.log_store', 1)) {
// Load the results from the log
$this->logresult = $this->get('Stats', 'log');
// Get the run time
$session = JFactory::getSession();
$runtime = $session->get('com_csvi.runtime');
if ($runtime > 0) {
$runtime = time() - $runtime;
}
$this->assignRef('runtime', JText::sprintf('COM_CSVI_RUNTIME_IMPORT', number_format($runtime / 60, 2), $runtime));
// Reset the run time
$session->set('com_csvi.runtime', null);
// Get the toolbar title
JToolBarHelper::title(JText::_('COM_CSVI_' . $this->logresult['action'] . '_RESULT'), 'csvi_' . $this->logresult['action'] . '_48');
} else {
$this->logresult = false;
}
// Get the panel
$this->loadHelper('panel');
// Display it all
parent::display($tpl);
}
/**
* Import the files
*
* @copyright
* @author RolandD
* @todo
* @see
* @access public
* @param
* @return string JSON encoded text
* @since 3.0
*/
public function display($tpl = null)
{
$jinput = JFactory::getApplication()->input;
if ($jinput->get('importsession', true, 'bool')) {
// Process the data
$this->get('ProcessData');
// Empty the message stack
$app = JFactory::getApplication();
$app->set('_messageQueue', array());
// Collect the results
$result = array();
// Set the view mode
if ($jinput->get('csvipreview', false, 'bool')) {
$result['view'] = 'preview';
$result['headers'] = $jinput->get('headers_preview', null, null);
$result['output'] = $jinput->get('data_preview', null, null);
if (empty($results['headers']) && empty($result['output'])) {
$result['process'] = false;
$csvilog = $jinput->get('csvilog', null, null);
$result['url'] = JURI::root() . 'administrator/index.php?option=com_csvi&task=process.finished&run_id=' . $csvilog->getId();
// Clean the session, nothing to import
$this->get('CleanSession');
} else {
$result['process'] = true;
}
} else {
$result['view'] = '';
// Get the number of records processed
$result['records'] = $jinput->get('recordsprocessed', 0, 'int');
if ($result['records'] == 0) {
$result['process'] = false;
$result['url'] = JURI::root() . 'administrator/index.php?option=com_csvi&task=process.finished&run_id=' . $jinput->get('run_id', 0, 'int');
} else {
$result['process'] = true;
}
}
} else {
$csvilog = $jinput->get('csvilog', null, null);
// Collect the results
$result = array();
$result['process'] = false;
$result['url'] = JURI::root() . 'administrator/index.php?option=com_csvi&task=process.finished&run_id=' . $jinput->get('run_id', 0, 'int');
// Clean the session, nothing to import
$this->get('CleanSession');
}
if ($result['process']) {
// Import is not finished, lets sleep
$settings = new CsviSettings();
sleep($settings->get('import.import_wait', 0));
}
// Output the results
echo json_encode($result);
}
/**
* Initialise some settings
*/
public function runCron()
{
// Buffer all output to prevent conflicts with external software
ob_start();
// Start the clock
$starttime = time();
$db = JFactory::getDbo();
// First check if we deal with a valid user
if ($this->Login()) {
// Set some global values
$jinput = JFactory::getApplication()->input;
$jfilter = new JFilterInput();
// Get the parameters
require_once JPATH_COMPONENT_ADMINISTRATOR . '/helpers/settings.php';
$settings = new CsviSettings();
// Check if we are running cron mode and set some necessary variables
$_SERVER['SERVER_ADDR'] = $_SERVER['HTTP_HOST'] = $settings->get('site.hostname');
$_SERVER['SCRIPT_NAME'] = '/index.php';
$_SERVER['REQUEST_URI'] = '/';
$_SERVER['PHP_SELF'] = '/index.php';
// Get the task to do
if (isset($this->_variables['task'])) {
$task = $jfilter->clean($this->_variables['task']);
} else {
$task = '';
}
// Perform the requested task
switch ($task) {
case 'maintenance':
$jinput->set('task', 'maintenance.' . $this->_variables['operation']);
// Fire CSVI VirtueMart
$this->ExecuteJob();
break;
default:
// Second check if any template is set to process
if (array_key_exists('template_id', $this->_variables)) {
$template_id = $jfilter->clean($this->_variables['template_id'], 'int');
} else {
$template_id = false;
}
if (array_key_exists('template_name', $this->_variables)) {
$template_name = $jfilter->clean($this->_variables['template_name']);
} else {
$template_name = false;
}
if ($template_id || $template_name) {
// There is a template_id or template name, get some details to streamline processing
$where = empty($template_id) ? 'name=' . $db->Quote($template_name) : 'id=' . $template_id;
// There is a template name, get some details to streamline processing
$q = "SELECT id AS template_id, name AS template_name, settings\r\n\t\t\t\t\t\t\tFROM #__csvi_template_settings\r\n\t\t\t\t\t\t\tWHERE " . $where;
$db->setQuery($q);
$row = $db->loadObject();
if (is_object($row)) {
echo JText::sprintf('COM_CSVI_PROCESSING_STARTED', date('jS F Y, g:i a')) . "\n";
echo JText::sprintf('COM_CSVI_TEMPLATE', $row->template_name) . "\n";
// Set the template ID
$jinput->set('select_template', $row->template_id);
$jinput->set('template_name', $row->template_name);
// Set the settings
if (array_key_exists('jform', $this->_variables)) {
$settings = CsviHelper::arrayExtend(json_decode($row->settings, true), $this->_variables['jform']);
} else {
$settings = json_decode($row->settings, true);
}
// Set some export settings
if ($settings['options']['action'] == 'export') {
// Export settings
$jinput->set('task', 'exportfile.process');
// Set export to
if ($settings['general']['exportto'] == 'todownload') {
$settings['general']['exportto'] = 'tofile';
}
} else {
if ($settings['options']['action'] == 'import') {
// Import settings
$jinput->set('task', 'importfile.doimport');
// Turn off preview
$settings['general']['show_preview'] = 0;
}
}
// Post the settings
$jinput->set('jform', $settings, 'post');
// Fire CSVI
$this->ExecuteJob();
} else {
if ($template_name) {
echo JText::sprintf('COM_CSVI_NO_TEMPLATE_FOUND', $template_name) . "\n";
} else {
if ($template_id) {
echo JText::sprintf('COM_CSVI_NO_TEMPLATE_FOUND', $template_id) . "\n";
}
}
}
} else {
echo JText::_('COM_CSVI_NO_TEMPLATE_SPECIFIED') . "\n";
}
break;
}
} else {
$error = JError::getError();
echo $error->message . "\n";
}
echo sprintf(JText::_('COM_CSVI_PROCESSING_FINISHED'), date('jS F Y, g:i a')) . "\n";
$duration = time() - $starttime;
if ($duration < 60) {
echo sprintf(JText::_('COM_CSVI_PROCESSING_SECONDS'), $duration) . "\n";
} else {
echo sprintf(JText::_('COM_CSVI_PROCESSING_MINUTES'), number_format($duration / 60, 2)) . "\n";
}
// Done, lets log the user out
$this->UserLogout();
// Display any generated messages
$messages = ob_get_contents();
@ob_end_clean();
echo $messages;
}
/**
* Start the import
*
* @copyright
* @author RolandD
* @todo Separate view for preview
* @todo Rewrite memory usage for debug
* @see
* @access public
* @param
* @return
* @since 3.0
*/
public function getProcessData()
{
$jinput = JFactory::getApplication()->input;
// Set some variables
$data_preview = array();
$processdata = true;
$redirect = false;
// Load the log
$csvilog = $jinput->get('csvilog', null, null);
// Load the settings
$settings = new CsviSettings();
// Load the template
$template = $jinput->get('template', null, null);
// Load the file
$csvifile = $jinput->get('csvifile', null, null);
// Set the table path
$this->addTablePath(JPATH_COMPONENT_ADMINISTRATOR . '/tables/' . $template->get('component', 'options'));
// Load the import routine
$classname = $this->getModel($template->get('operation', 'options'));
if (class_exists($classname)) {
$routine = new $classname();
} else {
$routine = false;
}
if ($routine) {
// Start processing data
while ($processdata) {
// Pass the total log line to the logger
$csvilog->setLinenumber($jinput->get('currentline', 0, 'int') + $jinput->get('totalline', 0, 'int'));
// If the number of lines is set to 0, do unlimited import
if ($settings->get('import.import_nolines', 0) == 0 || $jinput->get('cron', false, 'bool')) {
$nolines = $jinput->get('currentline', 0, 'int') + 1;
} else {
$nolines = $settings->get('import.import_nolines');
}
if ($jinput->get('currentline', 0, 'int') <= $nolines) {
// For XML files, it may be necessary to refresh the headers before reading the next record
if ($csvifile->extension == 'xml' && $template->get('refresh_xml_headers', 'general')) {
$csvifile->loadColumnHeaders();
if ($this->_retrieveConfigFields() == false) {
// Error found - Finish processing
$redirect = $this->finishProcess(false);
$processdata = false;
continue;
}
}
// Load the data
$this->csvi_data = $csvifile->ReadNextLine();
if ($this->csvi_data == false) {
if ($jinput->get('csvipreview', false, 'bool')) {
// Set the headers
$headers = array();
foreach ($this->_csvifields as $fieldname => $value) {
if ($value['published']) {
if (isset($routine->{$fieldname}) || empty($routine->{$fieldname})) {
$headers[] = $fieldname;
}
}
}
$jinput->set('headers_preview', $headers);
// Set the data
$jinput->set('data_preview', $data_preview);
// Clean the session
$this->getCleanPreview();
$processdata = false;
continue;
} else {
// Finish processing
$this->finishProcess(true);
$processdata = false;
}
} else {
// Check if we need to add any extra fields
if (count($this->_csvifields) > count($this->csvi_data)) {
foreach ($this->_csvifields as $fieldname => $details) {
if (!array_key_exists($details['order'], $this->csvi_data)) {
if (!empty($details['default_value'])) {
$this->csvi_data[$details['order']] = $details['default_value'];
}
}
}
// Check if the fields are now equal
if (count($this->_csvifields) > count($this->csvi_data)) {
$message = JText::sprintf('COM_CSVI_INCORRECT_COLUMN_COUNT', count($this->_csvifields), count($this->csvi_data));
$message .= '<br />' . JText::_('COM_CSVI_FIELDS') . '<br />';
$message .= '<table class="adminlist"><thead><tr><th>Position</th><th>Configuration</th><th>Import file</th></tr></thead><tfoot></tfoot>';
$message .= '<tbody>';
foreach ($this->_csvifields as $fieldname => $field_details) {
$message .= '<tr><td>' . $field_details['order'] . '</td><td>' . $fieldname . '</td><td>';
if (isset($this->csvi_data[$field_details['order']])) {
$message .= $this->csvi_data[$field_details['order']];
}
$message .= '</td></tr>';
}
$message .= '</tbody></table>';
$csvilog->AddStats('incorrect', $message, true);
// Finish processing
$this->finishProcess(true);
$processdata = false;
}
}
// Load ICEcat data if user wants to
$this->getIcecat();
// Validate the fields
$csvi_data = new JObject();
foreach ($this->_csvifields as $name => $details) {
if ($details['published']) {
$datafield = $this->validateInput($details['name'], $details['replace']);
if ($datafield !== false) {
// Check if we are dealing with the last field
if ($details == $this->_lastfield) {
$details['combine'] = false;
}
// See if we are combining the field
if ($details['combine']) {
$this->setCombineField($datafield, $name);
} else {
// Check if there are any fields to be combined
if (!empty($this->combine_fields)) {
// Get the fieldname the combine is for
$name = $this->combine_settings['fieldname'];
// Add the current data
$this->setCombineField($datafield);
// Get the combined data
$datafield = $this->getCombineField();
}
}
// Set the new value
$csvi_data->{$name} = $datafield;
}
}
}
$jinput->set('csvi_data', $csvi_data);
if ($this->_checkLimits()) {
// Notify the debug log what line we are one
$csvilog->addDebug(JText::sprintf('COM_CSVI_DEBUG_PROCESS_LINE', $jinput->get('currentline', 0, 'int') + $jinput->get('totalline', 0, 'int')));
// Start processing record
if ($routine->getStart()) {
if ($jinput->get('csvipreview', false, 'bool')) {
$this->loadSettings();
// Update preview data
foreach ($this->_csvifields as $fieldname => $value) {
if ($value['published']) {
if (isset($routine->{$fieldname})) {
$preview_data[$value['order']] = $routine->{$fieldname};
} else {
if (empty($routine->{$fieldname})) {
$preview_data[$value['order']] = '';
}
}
}
}
$data_preview[$jinput->get('currentline', 0, 'int')] = $preview_data;
if ($jinput->get('currentline', 0, 'int') == $settings->get('import.import_preview', 5)) {
// Set the headers
$headers = array();
foreach ($this->_csvifields as $fieldname => $value) {
if ($value['published']) {
if (isset($routine->{$fieldname}) || empty($routine->{$fieldname})) {
$headers[] = $fieldname;
}
}
}
$jinput->set('headers_preview', $headers);
// Set the data
$jinput->set('data_preview', $data_preview);
// Clean the session
$this->getCleanPreview();
$processdata = false;
continue;
}
} else {
// Now we import the rest of the records
$routine->getProcessRecord();
}
// Increase the number of records processed
$jinput->set('recordsprocessed', $jinput->get('recordsprocessed', 0, 'int') + 1);
} else {
// The routine reports a problem, usually unmet conditions
// Finish processing
$this->finishProcess(true);
// Stop from processing any further, no time left
$processdata = false;
}
// Increase linenumber
$jinput->set('currentline', $jinput->get('currentline', 0, 'int') + 1);
} else {
// Finish processing
$this->finishProcess(false);
// Stop from processing any further, no time left
$processdata = false;
}
}
} else {
// Finish processing
$this->finishProcess(false);
// Stop from processing any further, no time left
$processdata = false;
}
}
// Post Processing
if (method_exists($routine, 'getPostProcessing')) {
$routine->getPostProcessing(array_keys($this->_csvifields));
}
} else {
$csvilog->AddStats('incorrect', 'COM_CSVI_NO_VALID_CLASS_FOUND');
// Finish processing
$this->finishProcess(true);
}
}
/**
* Import the files
*
* @copyright
* @author RolandD
* @todo
* @see
* @access public
* @param
* @return string JSON encoded text
* @since 3.0
*/
public function display($tpl = null)
{
$jinput = JFactory::getApplication()->input;
if ($jinput->get('importsession', true, 'bool')) {
// Process the data
$this->get('ProcessData');
// Empty the message stack
$session = JFactory::getSession();
$session->set('application.queue', null);
// Collect the results
$result = array();
// Set the view mode
if ($jinput->get('csvipreview', false, 'bool')) {
$result['view'] = 'preview';
$result['headers'] = $jinput->get('headers_preview', null, null);
$result['output'] = $jinput->get('data_preview', null, null);
if (empty($results['headers']) && empty($result['output'])) {
$result['process'] = false;
$csvilog = $jinput->get('csvilog', null, null);
$result['url'] = JURI::root() . 'administrator/index.php?option=com_csvi&task=process.finished&run_id=' . $csvilog->getId();
// Clean the session, nothing to import
$this->get('CleanSession');
} else {
$result['process'] = true;
}
} else {
$result['view'] = '';
// Get the number of records processed
$result['records'] = $jinput->get('recordsprocessed', 0, 'int');
if ($result['records'] == 0 || $jinput->get('finished', false)) {
$result['process'] = false;
$result['url'] = JURI::root() . 'administrator/index.php?option=com_csvi&task=process.finished&run_id=' . $jinput->get('run_id', 0, 'int');
} else {
// Check if we are finished
$result['process'] = true;
}
}
} else {
// Collect the results
$result = array();
$result['process'] = false;
$result['url'] = JURI::root() . 'administrator/index.php?option=com_csvi&task=process.finished&run_id=' . $jinput->get('run_id', 0, 'int');
// Clean the session, nothing to import
$this->get('CleanSession');
}
// If the import is finished, call the plugins
if (!$result['process']) {
// Load the template
$session = JFactory::getSession();
$template = new CsviTemplate();
$template->load(unserialize($session->get('com_csvi.select_template')));
$options = array();
$options[] = $template->get('options');
$dispatcher = JDispatcher::getInstance();
JPluginHelper::importPlugin('csvi');
$dispatcher->trigger('onImportComplete', $options);
} else {
// Import is not finished, lets sleep
if ($jinput->get('currentline', 0, 'int') > 0 && !$jinput->get('finished', false)) {
$settings = new CsviSettings();
sleep($settings->get('import.import_wait', 0));
}
}
// Output the results
echo json_encode($result);
}
/**
* Request the data from ICEcat
*
* There are different URLs to get the data from:
*
* Open ICEcat users have access to:
* http://data.icecat.biz/export/freexml.int/INT/ for access to the standardized data files (QUALITY=ICECAT).
* The language-specific data-files are found here:
* http://data.icecat.biz/export/freexml.int/[code]/[product_id].xml, where [code] stands e.g. for NL, EN, FR, DE, IT, ES, DK etc.
*
* For the Full ICEcat subscribers, a separate directory structure is in place. The standardized files are located at:
* http://data.icecat.biz/export/level4/INT
* and the language dependent versions are found here:
* http://data.icecat.biz/export/level4/[code]/[product_id].xml, where [code] stands e.g. for NL, EN, FR, DE, IT, ES, DK, etc. For
*
* Products need to be matched to a product file found at http://data.icecat.biz/export/freexml/EN/
*
* an index file with references to all product data-sheets in ICEcat or Open ICEcat, also historical/obsolete products
* files.index.csv|xml or files.index.csv.gz|xml.gz
* a smaller index file with only references to the new or changed product data-sheets of the respective day
* daily.index.csv|xml or daily.index.csv.gz|xml.gz
* an index file with only the products that are currently on the market, as far as we can see that based on 100s of distributor and reseller price files
* on_market.index.csv|xml or on_market.index.csv.gz|xml.gz)
* an index file with the products that are or were on the market for which we only have basic market data, but no complete data-sheet
* nobody.index.csv|xml or nobody.index.csv.gz|xml.gz
*
* @copyright
* @author RolandD
* @todo Check for gzip functionality to reduce filesize
* @see
* @access private
* @param string $icecat_id the ICEcat ID to retrieve
* @return
* @since 3.0
*/
private function _callIcecat($icecat_id)
{
$jinput = JFactory::getApplication()->input;
$template = $jinput->get('template', null, null);
$csvilog = $jinput->get('csvilog', null, null);
$settings = new CsviSettings();
// Construct the URL
$url = $settings->get('icecat.ice_advanced') ? 'http://data.icecat.biz/export/level4/' : 'http://data.icecat.biz/export/freexml.int/';
// The language to use
$url .= $settings->get('icecat.ice_lang') . '/';
// The ID to retrieve
$url .= $icecat_id . '.xml';
$csvilog->addDebug(JText::sprintf('COM_CSVI_CALL_ICECAT_URL', $url));
// Initialise the curl call
$curl = curl_init();
// set URL and other appropriate options
curl_setopt($curl, CURLOPT_URL, $url);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
curl_setopt($curl, CURLOPT_USERPWD, $settings->get('icecat.ice_username') . ":" . $settings->get('icecat.ice_password'));
// grab URL and pass it to the browser
$this->_data = curl_exec($curl);
// close cURL resource, and free up system resources
curl_close($curl);
}
/**
* Start the import
*
* @copyright
* @author RolandD
* @todo Separate view for preview
* @todo Rewrite memory usage for debug
* @see
* @access public
* @param
* @return
* @since 3.0
*/
public final function getProcessData()
{
$jinput = JFactory::getApplication()->input;
// Set some variables
$data_preview = array();
$processdata = true;
$redirect = false;
// Load the log
$csvilog = $jinput->get('csvilog', null, null);
// Load the settings
$settings = new CsviSettings();
// Load the template
$template = $jinput->get('template', null, null);
// Load the file
$csvifile = $jinput->get('csvifile', null, null);
// Set the table path
$this->addTablePath(JPATH_COMPONENT_ADMINISTRATOR . '/tables/' . $template->get('component', 'options'));
// Load the import routine
$classname = $this->getModel($template->get('operation', 'options'));
if (class_exists($classname)) {
$routine = new $classname();
} else {
$routine = false;
}
if ($routine) {
// Start processing data
while ($processdata) {
// Pass the total log line to the logger
$csvilog->setLinenumber($jinput->get('currentline', 0, 'int') + $jinput->get('totalline', 0, 'int'));
// If the number of lines is set to 0, do unlimited import
if ($settings->get('import.import_nolines', 0) == 0 || $jinput->get('cron', false, 'bool')) {
$nolines = $jinput->get('currentline', 0, 'int') + 1;
} else {
$nolines = $settings->get('import.import_nolines');
}
if ($jinput->get('currentline', 0, 'int') <= $nolines) {
// Load the data
$result = $csvifile->ReadNextLine();
// Check the load result
if ($result == false) {
// Finish processing
$this->finishProcess(true);
$processdata = false;
} else {
// Load ICEcat data if user wants to
$this->_csvifields->getIcecat();
// Set the data to import
$this->_csvifields->prepareData();
if ($this->_checkLimits()) {
// Notify the debug log what line we are one
$csvilog->addDebug(JText::sprintf('COM_CSVI_DEBUG_PROCESS_LINE', $jinput->get('currentline', 0, 'int') + $jinput->get('totalline', 0, 'int')));
// Start processing record
if ($routine->getStart()) {
// Start processing the records
$routine->getProcessRecord();
// Increase the number of records processed
$jinput->set('recordsprocessed', $jinput->get('recordsprocessed', 0, 'int') + 1);
} else {
// The routine reports a problem, usually unmet conditions
// Finish processing
$this->finishProcess(true);
// Stop from processing any further, no time left
$processdata = false;
}
// Increase linenumber
$jinput->set('currentline', $jinput->get('currentline', 0, 'int') + 1);
} else {
// Finish processing
$this->finishProcess(false);
// Stop from processing any further, no time left
$processdata = false;
}
}
} else {
// Finish processing
$this->finishProcess(false);
// Stop from processing any further, no time left
$processdata = false;
}
// Clean the fields
$this->_csvifields->reset();
}
// Post Processing
if (method_exists($routine, 'getPostProcessing')) {
$routine->getPostProcessing($this->_csvifields->getFieldnames());
}
} else {
$csvilog->AddStats('incorrect', 'COM_CSVI_NO_VALID_CLASS_FOUND');
// Finish processing
$this->finishProcess(true);
}
}