<?php require_once 'Console/ProgressBar.php'; $bar = new Console_ProgressBar('* %fraction% [%bar%] %percent%', '=>', '-', 76, 7); for ($i = 0; $i <= 7; $i++) { sleep(1); $bar->update($i); } print "\n";
public function import(SplFileObject $file) { // We don't need all columns from imported CSV file, let's ignore them $ignoredColumns = $this->getIgnoredColumns($this->_city); // Get numer of lines in file $lineCount = $this->getCsvFileLineCount($file); // Show a progress bar if ($this->cli->options['verbose']) { $progressBar = new Console_ProgressBar('Importing: [%bar%] %percent%', '=', '.', '60', $lineCount - 1); $line = 0; } $lineImported = 0; // Iterate file by line foreach ($file as $row) { // We need to increase progressbar here -- because we filter out some lines later if ($this->cli->options['verbose']) $progressBar->update($line++); // Prepare associative array in format acceptable by fromArray() / synchronizeFromArray() $record = array_combine($this->_columns, $row); // We are interested in cities, villages etc. only if ($record['feature_class'] !== 'P') continue; // We want to skip ambadoned places, sections etc. if (preg_match('/^PPL[SRQX]$/s', $record['feature_code'])) continue; // Remove unnecessary columns foreach ($ignoredColumns as $ignoredColumn) unset($record[$ignoredColumn]); // Transform empty strings onto NULL values $record = array_map(function($item) { return ($item === '' ? null : $item); }, $record); // Skip ill-formed records $skip = false; foreach ($record as $field => $value) { // Only admin2_code field can be NULL if ($value === null && $field !== 'admin2_code') $skip = true; } if ($skip === true) continue; // Save record to the database $city = Doctrine_Core::getTable($this->_cityComponentName)->find($record['city_id']); $city = ($city instanceof $this->_cityComponentName ? $city : clone $this->_city); $city->fromArray($record); $city->save(); $city->free(); // Increase number of imported records $lineImported++; } // Fix some incorrect data we know of $this->fixIncorrectRecords(); printf("\nSuccessfully imported %d out of %d records.\n", $lineImported, $lineCount); }
// Add some spacing below the progress bar. echo PHP_EOL; /*%******************************************************************************************%*/ // UPLOAD SAMPLE FILE TO S3 $_100_percent = 0; // Create a bucket to upload to $bucket = 's3-progress-bar-' . strtolower($s3->key); if (!$s3->if_bucket_exists($bucket)) { $response = $s3->create_bucket($bucket, AmazonS3::REGION_US_E1); if (!$response->isOK()) { die('Could not create `' . $bucket . '`.'); } } // Instantiate a new progress bar. // We won't know the max number of bytes until the download starts, so we'll handle that in our callback. $progress_bar = new Console_ProgressBar('* %fraction% KB [%bar%] %percent%', '=>', ' ', 100, 1); $progress_bar->UPDATED = false; // Register a callback function to execute when a stream is written locally. $s3->register_streaming_read_callback('read_callback'); function read_callback($curl_handle, $file_handle, $length) { // Import from global scope $progress_bar = $GLOBALS['progress_bar']; // Have we updated the format with updated information yet? if (!$progress_bar->UPDATED) { // Store the total size in local & global scope $_100_percent = $GLOBALS['_100_percent'] = curl_getinfo($curl_handle, CURLINFO_CONTENT_LENGTH_UPLOAD); // Add the Content-Length of the file as the max number of bytes. $progress_bar->reset('* %fraction% KB [%bar%] %percent%', '=>', ' ', 100, $_100_percent); $progress_bar->UPDATED = true; }
/** * @param $fileName * @param $tableName * @param $importFieldsType * @param $fieldsList * @param $csvSeparator * @param $exportFieldsName */ private function runCSVExport($fileName, $tableName, $importFieldsType, $fieldsList, $csvSeparator, $exportFieldsName) { $fileName = $fileName . '.csv'; $fp = fopen($fileName, 'w'); if ($exportFieldsName) { FileManager::putCSV($fp, $fieldsList, $csvSeparator, ''); } if ($importFieldsType == "all") { $query = "SELECT * FROM {$tableName}"; } else { $query = "SELECT " . implode(",", $fieldsList) . " FROM {$tableName}"; } $format = "Exported rows : %current%/%max% [%bar%] %percent% (%elapsed%)"; $filler = '=>'; $empty = ' '; $width = 100; $cp = 0; $result = $this->DB->Execute($query); if ($result !== false) { $count = $result->_numOfRows; /** * Init Progress Bar */ $bar = new Console_ProgressBar($format, $filler, $empty, $width, $count); while (!$result->EOF) { $aRecords = $result->fields; FileManager::putCSV($fp, $aRecords, $csvSeparator, ''); $result->MoveNext(); $cp++; $bar->update($cp); } $this->showNotice("Export has been successfully finished : " . $cp . " rows exported."); } fclose($fp); $errorMsg = $this->DB->ErrorMsg(); if ($errorMsg != "") { $this->showError($errorMsg); } }
/** * launch_sending() * * Cette fonction est appellée soit dans envoi.php lors de l'envoi, soit * dans le fichier appellé originellement cron.php * * @param array $listdata Tableau des données de la liste concernée * @param array $logdata Tableau des données de la newsletter * @param array $supp_address Adresses de destinataires supplémentaires * * @return string */ function launch_sending($listdata, $logdata, $supp_address = array()) { global $nl_config, $db, $lang, $other_tags; // // On commence par poser un verrou sur un fichier lock, // il ne faut pas qu'il y ait simultanément plusieurs flôts d'envois // pour une même liste de diffusion. // $lockfile = sprintf(WA_LOCKFILE, $listdata['liste_id']); if (file_exists($lockfile)) { $isBeginning = false; $fp = fopen($lockfile, 'r+'); $supp_address = array(); // On en tient pas compte, ça l'a déjà été lors du premier flôt } else { $isBeginning = true; $fp = fopen($lockfile, 'w'); @chmod($lockfile, 0600); } if (!flock($fp, LOCK_EX | LOCK_NB)) { fclose($fp); return $lang['Message']['List_is_busy']; } if (filesize($lockfile) > 0) { // // L'envoi a planté au cours d'un "flôt" précédent. On récupère les éventuels // identifiants d'abonnés stockés dans le fichier lock et on met à jour la table // $abo_ids = fread($fp, filesize($lockfile)); $abo_ids = array_map('trim', explode("\n", trim($abo_ids))); if (count($abo_ids) > 0) { $abo_ids = array_unique(array_map('intval', $abo_ids)); $sql = "UPDATE " . ABO_LISTE_TABLE . "\n\t\t\t\tSET send = 1\n\t\t\t\tWHERE abo_id IN(" . implode(', ', $abo_ids) . ")\n\t\t\t\t\tAND liste_id = " . $listdata['liste_id']; if (!$db->query($sql)) { trigger_error('Impossible de mettre à jour la table des abonnés', ERROR); } } ftruncate($fp, 0); fseek($fp, 0); } // // Initialisation de la classe mailer // require WAMAILER_DIR . '/class.mailer.php'; $mailer = new Mailer(WA_ROOTDIR . '/language/email_' . $nl_config['language'] . '/'); if ($nl_config['use_smtp']) { $mailer->smtp_path = WAMAILER_DIR . '/'; $mailer->use_smtp($nl_config['smtp_host'], $nl_config['smtp_port'], $nl_config['smtp_user'], $nl_config['smtp_pass']); } $mailer->set_charset($lang['CHARSET']); $mailer->set_from($listdata['sender_email'], unhtmlspecialchars($listdata['liste_name'])); if ($listdata['return_email'] != '') { $mailer->set_return_path($listdata['return_email']); } // // On traite les données de la newsletter à envoyer // if (preg_match('/[\\x80-\\x9F]/', $logdata['log_subject']) || preg_match('/[\\x80-\\x9F]/', $logdata['log_body_text']) || preg_match('/[\\x80-\\x9F]/', $logdata['log_body_html'])) { if (TRANSLITE_INVALID_CHARS == false) { $logdata['log_subject'] = wan_utf8_encode($logdata['log_subject']); $logdata['log_body_text'] = wan_utf8_encode($logdata['log_body_text']); $logdata['log_body_html'] = wan_utf8_encode($logdata['log_body_html']); $lang['Label_link'] = wan_utf8_encode($lang['Label_link']); $mailer->set_charset('UTF-8'); } else { $logdata['log_subject'] = purge_latin1($logdata['log_subject'], true); $logdata['log_body_text'] = purge_latin1($logdata['log_body_text'], true); $logdata['log_body_html'] = purge_latin1($logdata['log_body_html']); } } $mailer->set_subject($logdata['log_subject']); $body = array(FORMAT_TEXTE => $logdata['log_body_text'], FORMAT_HTML => $logdata['log_body_html']); // // Ajout du lien de désinscription, selon les méthodes d'envoi/format utilisés // $link = newsletter_links($listdata); if ($listdata['use_cron'] || $nl_config['engine_send'] == ENGINE_BCC) { $body[FORMAT_TEXTE] = str_replace('{LINKS}', $link[FORMAT_TEXTE], $body[FORMAT_TEXTE]); $body[FORMAT_HTML] = str_replace('{LINKS}', $link[FORMAT_HTML], $body[FORMAT_HTML]); } // // On s'occupe maintenant des fichiers joints ou incorporés // Si les fichiers sont stockés sur un serveur ftp, on les rapatrie le temps du flot d'envoi // $total_files = count($logdata['joined_files']); $tmp_files = array(); require WA_ROOTDIR . '/includes/class.attach.php'; $attach = new Attach(); hasCidReferences($body[FORMAT_HTML], $refs); for ($i = 0; $i < $total_files; $i++) { $real_name = $logdata['joined_files'][$i]['file_real_name']; $physical_name = $logdata['joined_files'][$i]['file_physical_name']; $mime_type = $logdata['joined_files'][$i]['file_mimetype']; $error = FALSE; $msg = array(); $attach->joined_file_exists($physical_name, $error, $msg); if ($error) { $error = FALSE; continue; } if ($nl_config['use_ftp']) { $file_path = $attach->ftp_to_tmp($logdata['joined_files'][$i]); array_push($tmp_files, $file_path); } else { $file_path = WA_ROOTDIR . '/' . $nl_config['upload_path'] . $physical_name; } if (is_array($refs) && in_array($real_name, $refs)) { $embedded = TRUE; } else { $embedded = FALSE; } $mailer->attachment($file_path, $real_name, 'attachment', $mime_type, $embedded); } // // Récupération des champs des tags personnalisés // if (count($other_tags) > 0) { $fields_str = ''; foreach ($other_tags as $data) { $fields_str .= 'a.' . $data['column_name'] . ', '; } } else { $fields_str = ''; } // // Si on en est au premier flôt, on récupère également les adresses email // des administrateurs ayant activés l'option de réception de copie // if ($isBeginning) { $sql = "SELECT a.admin_email\n\t\t\tFROM " . ADMIN_TABLE . " AS a\n\t\t\t\tINNER JOIN " . AUTH_ADMIN_TABLE . " AS aa ON aa.admin_id = a.admin_id\n\t\t\t\t\tAND aa.cc_admin = " . TRUE; if (!($result = $db->query($sql))) { trigger_error('Impossible d\'obtenir la liste des fichiers joints', ERROR); } while ($email = $result->column('admin_email')) { array_push($supp_address, $email); } $result->free(); $supp_address = array_unique($supp_address); // Au cas où... } // // On récupère les infos sur les abonnés destinataires // $sql = "SELECT COUNT(a.abo_id) AS total\n\t\tFROM " . ABONNES_TABLE . " AS a\n\t\t\tINNER JOIN " . ABO_LISTE_TABLE . " AS al ON al.abo_id = a.abo_id\n\t\t\t\tAND al.liste_id = {$listdata['liste_id']}\n\t\t\t\tAND al.confirmed = " . SUBSCRIBE_CONFIRMED . "\n\t\t\t\tAND al.send = 0\n\t\tWHERE a.abo_status = " . ABO_ACTIF; if (!($result = $db->query($sql))) { trigger_error('Impossible d\'obtenir le nombre d\'adresses emails', ERROR); } $total_abo = $result->column('total'); if ($nl_config['emails_sended'] > 0) { $total_abo = min($total_abo, $nl_config['emails_sended']); } $sql = "SELECT a.abo_id, a.abo_pseudo, {$fields_str} a.abo_email, al.register_key, al.format\n\t\tFROM " . ABONNES_TABLE . " AS a\n\t\t\tINNER JOIN " . ABO_LISTE_TABLE . " AS al ON al.abo_id = a.abo_id\n\t\t\t\tAND al.liste_id = {$listdata['liste_id']}\n\t\t\t\tAND al.confirmed = " . SUBSCRIBE_CONFIRMED . "\n\t\t\t\tAND al.send = 0\n\t\tWHERE a.abo_status = " . ABO_ACTIF; if ($nl_config['emails_sended'] > 0) { $sql .= " LIMIT {$nl_config['emails_sended']} OFFSET 0"; } if (!($result = $db->query($sql))) { trigger_error('Impossible d\'obtenir la liste des adresses emails', ERROR); } $abo_ids = array(); $format = $listdata['liste_format'] != FORMAT_MULTIPLE ? $listdata['liste_format'] : false; if ($row = $result->fetch()) { if ($nl_config['engine_send'] == ENGINE_BCC) { fake_header(false); $abonnes = array(FORMAT_TEXTE => array(), FORMAT_HTML => array()); $abo_ids = array(FORMAT_TEXTE => array(), FORMAT_HTML => array()); do { $abo_format = !$format ? $row['format'] : $format; array_push($abo_ids[$abo_format], $row['abo_id']); array_push($abonnes[$abo_format], $row['abo_email']); fake_header(true); } while ($row = $result->fetch()); if ($listdata['liste_format'] != FORMAT_HTML) { $abonnes[FORMAT_TEXTE] = array_merge($abonnes[FORMAT_TEXTE], $supp_address); } if ($listdata['liste_format'] != FORMAT_TEXTE) { $abonnes[FORMAT_HTML] = array_merge($abonnes[FORMAT_HTML], $supp_address); } // // Tableau pour remplacer les tags par des chaines vides // Non utilisation des tags avec le moteur d'envoi en copie cachée // $tags_replace = array('NAME' => ''); if (count($other_tags) > 0) { foreach ($other_tags as $data) { $tags_replace[$data['tag_name']] = ''; } } if (count($abonnes[FORMAT_TEXTE]) > 0) { $mailer->set_address($abonnes[FORMAT_TEXTE], 'Bcc'); $mailer->set_format(FORMAT_TEXTE); $mailer->set_message($body[FORMAT_TEXTE]); $mailer->assign_tags($tags_replace); if (!$mailer->send()) { trigger_error(sprintf($lang['Message']['Failed_sending2'], $mailer->msg_error), ERROR); } fwrite($fp, implode("\n", $abo_ids[FORMAT_TEXTE]) . "\n"); } $mailer->clear_address(); if (count($abonnes[FORMAT_HTML]) > 0) { $mailer->set_address($abonnes[FORMAT_HTML], 'Bcc'); $mailer->set_format($listdata['liste_format']); $mailer->assign_tags($tags_replace); $mailer->set_message($body[FORMAT_HTML]); if ($listdata['liste_format'] == FORMAT_MULTIPLE) { $mailer->set_altmessage($body[FORMAT_TEXTE]); } if (!$mailer->send()) { trigger_error(sprintf($lang['Message']['Failed_sending2'], $mailer->msg_error), ERROR); } fwrite($fp, implode("\n", $abo_ids[FORMAT_HTML]) . "\n"); } $abo_ids = array_merge($abo_ids[FORMAT_TEXTE], $abo_ids[FORMAT_HTML]); } else { if ($nl_config['engine_send'] == ENGINE_UNIQ) { if (defined('IN_COMMANDLINE')) { require WA_ROOTDIR . '/contrib/Console/ProgressBar.php'; // // Initialisation de la barre de progression des envois // $bar = new Console_ProgressBar('Sending emails %percent% [%bar%] %current% of %max%', '=>', ' ', 80, $total_abo + count($supp_address), array('ansi_terminal' => ANSI_TERMINAL)); } else { fake_header(false); } if (($isPHP5 = version_compare(phpversion(), '5.0.0', '>=')) == true) { eval('$mailerText = clone $mailer;'); eval('$mailerHTML = clone $mailer;'); } else { $mailerText = $mailer; $mailerHTML = $mailer; } if (!$listdata['use_cron']) { $body[FORMAT_TEXTE] = str_replace('{LINKS}', $link[FORMAT_TEXTE], $body[FORMAT_TEXTE]); $body[FORMAT_HTML] = str_replace('{LINKS}', $link[FORMAT_HTML], $body[FORMAT_HTML]); } $mailerText->set_format(FORMAT_TEXTE); $mailerText->set_message($body[FORMAT_TEXTE]); $mailerHTML->set_format(FORMAT_HTML); if ($listdata['liste_format'] == FORMAT_MULTIPLE) { $mailerHTML->set_format(FORMAT_MULTIPLE); $mailerHTML->set_altmessage($body[FORMAT_TEXTE]); } $mailerHTML->set_message($body[FORMAT_HTML]); $supp_address_ok = array(); foreach ($supp_address as $address) { if ($listdata['liste_format'] != FORMAT_HTML) { array_push($supp_address_ok, array('format' => FORMAT_TEXTE, 'abo_pseudo' => '', 'abo_email' => $address, 'register_key' => '', 'abo_id' => -1)); } if ($listdata['liste_format'] != FORMAT_TEXTE) { array_push($supp_address_ok, array('format' => FORMAT_HTML, 'abo_pseudo' => '', 'abo_email' => $address, 'register_key' => '', 'abo_id' => -1)); } } $counter = 0; $sendError = 0; do { $counter++; $abo_format = !$format ? $row['format'] : $format; if ($abo_format == FORMAT_TEXTE) { if ($isPHP5 == true) { eval('$mailer = clone $mailerText;'); } else { $mailer = $mailerText; } } else { if ($isPHP5 == true) { eval('$mailer = clone $mailerHTML;'); } else { $mailer = $mailerHTML; } } if ($row['abo_pseudo'] != '') { $address = array($row['abo_pseudo'] => $row['abo_email']); } else { $address = $row['abo_email']; } $mailer->clear_address(); $mailer->set_address($address); // // Traitement des tags et tags personnalisés // $tags_replace = array(); if ($row['abo_pseudo'] != '') { $tags_replace['NAME'] = $abo_format == FORMAT_HTML ? $row['abo_pseudo'] : unhtmlspecialchars($row['abo_pseudo']); } else { $tags_replace['NAME'] = ''; } if (count($other_tags) > 0) { foreach ($other_tags as $data) { if (isset($row[$data['column_name']])) { if (!is_numeric($row[$data['column_name']]) && $abo_format == FORMAT_HTML) { $row[$data['column_name']] = htmlspecialchars($row[$data['column_name']]); } $tags_replace[$data['tag_name']] = $row[$data['column_name']]; continue; } $tags_replace[$data['tag_name']] = ''; } } if (!$listdata['use_cron']) { $tags_replace = array_merge($tags_replace, array('WA_CODE' => $row['register_key'], 'WA_EMAIL' => rawurlencode($row['abo_email']))); } $mailer->assign_tags($tags_replace); // envoi if (!$mailer->send()) { $sendError++; } if ($row['abo_id'] != -1) { array_push($abo_ids, $row['abo_id']); fwrite($fp, "{$row['abo_id']}\n"); } if (defined('IN_COMMANDLINE')) { $bar->update($counter); if (SEND_DELAY > 0 && $counter % SEND_PACKET == 0) { sleep(SEND_DELAY); } } else { fake_header(true); } } while (($row = $result->fetch()) || ($row = array_pop($supp_address_ok)) != null); // // Aucun email envoyé, il y a manifestement un problème, on affiche le message d'erreur // if ($sendError == $total_abo) { flock($fp, LOCK_UN); fclose($fp); unlink($lockfile); trigger_error(sprintf($lang['Message']['Failed_sending2'], $mailer->msg_error), ERROR); } } else { trigger_error('Unknown_engine', ERROR); } } $result->free(); } else { if ($isBeginning) { // // Aucun abonné dont le champ send soit positionné à 0 et nous sommes au // début de l'envoi. Cette liste ne comporte donc pas encore d'abonné. // return $lang['Message']['No_subscribers']; } } // // Si l'option FTP est utilisée, suppression des fichiers temporaires // if ($nl_config['use_ftp']) { foreach ($tmp_files as $filename) { $attach->remove_file($filename); } } unset($tmp_files); $no_send = $sended = 0; if (!$db->ping()) { // // L'envoi a duré trop longtemps et la connexion au serveur SQL a été perdue // if (SQL_DRIVER == 'mysqli') { trigger_error("La connexion à la base de données a été perdue.<br />\nVous devriez mettre l'option PHP mysqli.reconnect à On dans le php.ini,<br />\npour permettre la reconnexion automatique au serveur.", ERROR); } else { trigger_error("La connexion à la base de données a été perdue", ERROR); } } if (count($abo_ids) > 0) { $sql = "UPDATE " . ABO_LISTE_TABLE . "\n\t\t\tSET send = 1\n\t\t\tWHERE abo_id IN(" . implode(', ', $abo_ids) . ")\n\t\t\t\tAND liste_id = " . $listdata['liste_id']; if (!$db->query($sql)) { trigger_error('Impossible de mettre à jour la table des abonnés (connexion au serveur sql perdue)', ERROR); } } $sql = "SELECT COUNT(*) AS num_dest, al.send\n\t\tFROM " . ABO_LISTE_TABLE . " AS al\n\t\t\tINNER JOIN " . ABONNES_TABLE . " AS a ON a.abo_id = al.abo_id\n\t\t\t\tAND a.abo_status = " . ABO_ACTIF . "\n\t\tWHERE al.liste_id = {$listdata['liste_id']}\n\t\t\tAND al.confirmed = " . SUBSCRIBE_CONFIRMED . "\n\t\tGROUP BY al.send"; if (!($result = $db->query($sql))) { trigger_error('Impossible d\'obtenir le nombre d\'envois restants à faire', ERROR); } while ($row = $result->fetch()) { if ($row['send'] == 1) { $sended = $row['num_dest']; } else { $no_send = $row['num_dest']; } } $result->free(); ftruncate($fp, 0); flock($fp, LOCK_UN); fclose($fp); if ($no_send > 0) { $message = sprintf($lang['Message']['Success_send'], $nl_config['emails_sended'], $sended, $sended + $no_send); if (!defined('IN_COMMANDLINE')) { if (!empty($_GET['step']) && $_GET['step'] == 'auto') { Location("envoi.php?mode=progress&id={$logdata['log_id']}&step=auto"); } $message .= '<br /><br />' . sprintf($lang['Click_resend_auto'], '<a href="' . sessid('./envoi.php?mode=progress&id=' . $logdata['log_id'] . '&step=auto') . '">', '</a>'); $message .= '<br /><br />' . sprintf($lang['Click_resend_manuel'], '<a href="' . sessid('./envoi.php?mode=progress&id=' . $logdata['log_id']) . '">', '</a>'); } } else { unlink($lockfile); $db->beginTransaction(); $sql = "UPDATE " . LOG_TABLE . "\n\t\t\tSET log_status = " . STATUS_SENDED . ",\n\t\t\t\tlog_numdest = {$sended}\n\t\t\tWHERE log_id = " . $logdata['log_id']; if (!$db->query($sql)) { trigger_error('Impossible de mettre à jour la table des logs', ERROR); } $sql = "UPDATE " . ABO_LISTE_TABLE . "\n\t\t\tSET send = 0\n\t\t\tWHERE liste_id = " . $listdata['liste_id']; if (!$db->query($sql)) { trigger_error('Impossible de mettre à jour la table des abonnés', ERROR); } $sql = "UPDATE " . LISTE_TABLE . "\n\t\t\tSET liste_numlogs = liste_numlogs + 1\n\t\t\tWHERE liste_id = " . $listdata['liste_id']; if (!$db->query($sql)) { trigger_error('Impossible de mettre à jour la table des listes', ERROR); } $db->commit(); $message = sprintf($lang['Message']['Success_send_finish'], $sended); } return $message; }