/** * Performs a batch insert into a specific table using either LOAD DATA INFILE or plain INSERTs, * as a fallback. On MySQL, LOAD DATA INFILE is 20x faster than a series of plain INSERTs. * * @param string $tableName PREFIXED table name! you must call Common::prefixTable() before passing the table name * @param array $fields array of unquoted field names * @param array $values array of data to be inserted * @param bool $throwException Whether to throw an exception that was caught while trying * LOAD DATA INFILE, or not. * @throws Exception * @return bool True if the bulk LOAD was used, false if we fallback to plain INSERTs */ public static function tableInsertBatch($tableName, $fields, $values, $throwException = false) { $filePath = PIWIK_USER_PATH . '/tmp/assets/' . $tableName . '-' . Common::generateUniqId() . '.csv'; $filePath = SettingsPiwik::rewriteTmpPathWithInstanceId($filePath); $loadDataInfileEnabled = Config::getInstance()->General['enable_load_data_infile']; if ($loadDataInfileEnabled && Db::get()->hasBulkLoader()) { try { $fileSpec = array('delim' => "\t", 'quote' => '"', 'escape' => '\\\\', 'escapespecial_cb' => function ($str) { return str_replace(array(chr(92), chr(34)), array(chr(92) . chr(92), chr(92) . chr(34)), $str); }, 'eol' => "\r\n", 'null' => 'NULL'); // hack for charset mismatch if (!DbHelper::isDatabaseConnectionUTF8() && !isset(Config::getInstance()->database['charset'])) { $fileSpec['charset'] = 'latin1'; } self::createCSVFile($filePath, $fileSpec, $values); if (!is_readable($filePath)) { throw new Exception("File {$filePath} could not be read."); } $rc = self::createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec); if ($rc) { unlink($filePath); return true; } } catch (Exception $e) { Log::info("LOAD DATA INFILE failed or not supported, falling back to normal INSERTs... Error was: %s", $e->getMessage()); if ($throwException) { throw $e; } } } // if all else fails, fallback to a series of INSERTs @unlink($filePath); self::tableInsertBatchIterate($tableName, $fields, $values); return false; }
public static function update() { $salt = Common::generateUniqId(); $config = Config::getInstance(); $superuser = $config->superuser; if (!isset($superuser['salt'])) { try { if (is_writable(Config::getLocalConfigPath())) { $superuser['salt'] = $salt; $config->superuser = $superuser; $config->forceSave(); } else { throw new \Exception('mandatory update failed'); } } catch (\Exception $e) { throw new \Piwik\UpdaterErrorException("Please edit your config/config.ini.php file and add below <code>[superuser]</code> the following line: <br /><code>salt = {$salt}</code>"); } } $plugins = $config->Plugins; if (!in_array('MultiSites', $plugins)) { try { if (is_writable(Config::getLocalConfigPath())) { $plugins[] = 'MultiSites'; $config->Plugins = $plugins; $config->forceSave(); } else { throw new \Exception('optional update failed'); } } catch (\Exception $e) { throw new \Exception("You can now enable the new MultiSites plugin in the Plugins screen in the Piwik admin!"); } } Updater::updateDatabase(__FILE__, self::getSql()); }
/** * Performs a batch insert into a specific table using either LOAD DATA INFILE or plain INSERTs, * as a fallback. On MySQL, LOAD DATA INFILE is 20x faster than a series of plain INSERTs. * * @param string $tableName PREFIXED table name! you must call Common::prefixTable() before passing the table name * @param array $fields array of unquoted field names * @param array $values array of data to be inserted * @param bool $throwException Whether to throw an exception that was caught while trying * LOAD DATA INFILE, or not. * @param string $charset The charset to use, defaults to utf8 * @throws Exception * @return bool True if the bulk LOAD was used, false if we fallback to plain INSERTs */ public static function tableInsertBatch($tableName, $fields, $values, $throwException = false, $charset = 'utf8') { $loadDataInfileEnabled = Config::getInstance()->General['enable_load_data_infile']; if ($loadDataInfileEnabled && Db::get()->hasBulkLoader()) { $path = self::getBestPathForLoadData(); $filePath = $path . $tableName . '-' . Common::generateUniqId() . '.csv'; try { $fileSpec = array('delim' => "\t", 'quote' => '"', 'escape' => '\\\\', 'escapespecial_cb' => function ($str) { return str_replace(array(chr(92), chr(34)), array(chr(92) . chr(92), chr(92) . chr(34)), $str); }, 'eol' => "\r\n", 'null' => 'NULL', 'charset' => $charset); self::createCSVFile($filePath, $fileSpec, $values); if (!is_readable($filePath)) { throw new Exception("File {$filePath} could not be read."); } $rc = self::createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec); if ($rc) { unlink($filePath); return true; } } catch (Exception $e) { if ($throwException) { throw $e; } } // if all else fails, fallback to a series of INSERTs if (file_exists($filePath)) { @unlink($filePath); } } self::tableInsertBatchIterate($tableName, $fields, $values); return false; }
private static function migrateConfigSuperUserToDb() { $config = Config::getInstance(); if (!$config->existsLocalConfig()) { return; } try { $superUser = $config->superuser; } catch (\Exception $e) { $superUser = null; } if (!empty($superUser['bridge']) || empty($superUser) || empty($superUser['login'])) { // there is a super user which is not from the config but from the bridge, that means we already have // a super user in the database return; } $userApi = UsersManagerApi::getInstance(); try { Db::get()->insert(Common::prefixTable('user'), array('login' => $superUser['login'], 'password' => $superUser['password'], 'alias' => $superUser['login'], 'email' => $superUser['email'], 'token_auth' => $userApi->getTokenAuth($superUser['login'], $superUser['password']), 'date_registered' => Date::now()->getDatetime(), 'superuser_access' => 1)); } catch (\Exception $e) { echo "There was an issue, but we proceed: " . $e->getMessage(); } if (array_key_exists('salt', $superUser)) { $salt = $superUser['salt']; } else { $salt = Common::generateUniqId(); } $config->General['salt'] = $salt; $config->superuser = array(); $config->forceSave(); }
/** * Performs a batch insert into a specific table using either LOAD DATA INFILE or plain INSERTs, * as a fallback. On MySQL, LOAD DATA INFILE is 20x faster than a series of plain INSERTs. * * @param string $tableName PREFIXED table name! you must call Common::prefixTable() before passing the table name * @param array $fields array of unquoted field names * @param array $values array of data to be inserted * @param bool $throwException Whether to throw an exception that was caught while trying * LOAD DATA INFILE, or not. * @throws Exception * @return bool True if the bulk LOAD was used, false if we fallback to plain INSERTs */ public static function tableInsertBatch($tableName, $fields, $values, $throwException = false) { $filePath = StaticContainer::get('path.tmp') . '/assets/' . $tableName . '-' . Common::generateUniqId() . '.csv'; $loadDataInfileEnabled = Config::getInstance()->General['enable_load_data_infile']; if ($loadDataInfileEnabled && Db::get()->hasBulkLoader()) { try { $fileSpec = array('delim' => "\t", 'quote' => '"', 'escape' => '\\\\', 'escapespecial_cb' => function ($str) { return str_replace(array(chr(92), chr(34)), array(chr(92) . chr(92), chr(92) . chr(34)), $str); }, 'eol' => "\r\n", 'null' => 'NULL'); // hack for charset mismatch if (!DbHelper::isDatabaseConnectionUTF8() && !isset(Config::getInstance()->database['charset'])) { $fileSpec['charset'] = 'latin1'; } self::createCSVFile($filePath, $fileSpec, $values); if (!is_readable($filePath)) { throw new Exception("File {$filePath} could not be read."); } $rc = self::createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec); if ($rc) { unlink($filePath); return true; } } catch (Exception $e) { if ($throwException) { throw $e; } } } // if all else fails, fallback to a series of INSERTs @unlink($filePath); self::tableInsertBatchIterate($tableName, $fields, $values); return false; }
public function acquireLock($id) { $this->lockKey = $this->lockKeyStart . $id; $lockValue = substr(Common::generateUniqId(), 0, 12); $locked = $this->backend->setIfNotExists($this->lockKey, $lockValue, $ttlInSeconds = 60); if ($locked) { $this->lockValue = $lockValue; } return $locked; }
public function __invoke(array $record) { if (Common::isPhpCliMode()) { return $record; } if (empty($this->currentRequestKey)) { $this->currentRequestKey = substr(Common::generateUniqId(), 0, 5); } $record['extra']['request_id'] = $this->currentRequestKey; return $record; }
/** * Returns an existing nonce by ID. If none exists, a new nonce will be generated. * * @param string $id Unique id to avoid namespace conflicts, e.g., `'ModuleName.ActionName'`. * @param int $ttl Optional time-to-live in seconds; default is 5 minutes. (ie, in 5 minutes, * the nonce will no longer be valid). * @return string */ public static function getNonce($id, $ttl = 600) { // save session-dependent nonce $ns = new SessionNamespace($id); $nonce = $ns->nonce; // re-use an unexpired nonce (a small deviation from the "used only once" principle, so long as we do not reset the expiration) // to handle browser pre-fetch or double fetch caused by some browser add-ons/extensions if (empty($nonce)) { // generate a new nonce $nonce = md5(SettingsPiwik::getSalt() . time() . Common::generateUniqId()); $ns->nonce = $nonce; } // extend lifetime if nonce is requested again to prevent from early timeout if nonce is requested again // a few seconds before timeout $ns->setExpirationSeconds($ttl, 'nonce'); return $nonce; }
public function installOrUpdatePluginFromFile($pathToZip) { $tmpPluginName = 'uploaded' . Common::generateUniqId(); $tmpPluginFolder = StaticContainer::get('path.tmp') . self::PATH_TO_DOWNLOAD . $tmpPluginName; try { $this->makeSureFoldersAreWritable(); $this->extractPluginFiles($pathToZip, $tmpPluginFolder); $this->makeSurePluginJsonExists($tmpPluginFolder); $metadata = $this->getPluginMetadataIfValid($tmpPluginFolder); $this->makeSureThereAreNoMissingRequirements($metadata); $this->pluginName = $metadata->name; $this->fixPluginFolderIfNeeded($tmpPluginFolder); $this->copyPluginToDestination($tmpPluginFolder); Filesystem::deleteAllCacheOnUpdate($this->pluginName); } catch (\Exception $e) { $this->removeFileIfExists($pathToZip); $this->removeFolderIfExists($tmpPluginFolder); throw $e; } $this->removeFileIfExists($pathToZip); $this->removeFolderIfExists($tmpPluginFolder); return $metadata; }
/** * Write configuration file from session-store */ private function createConfigFile($dbInfos) { $config = Config::getInstance(); // make sure DB sessions are used if the filesystem is NFS if (Filesystem::checkIfFileSystemIsNFS()) { $config->General['session_save_handler'] = 'dbtable'; } if (count($headers = ProxyHeaders::getProxyClientHeaders()) > 0) { $config->General['proxy_client_headers'] = $headers; } if (count($headers = ProxyHeaders::getProxyHostHeaders()) > 0) { $config->General['proxy_host_headers'] = $headers; } if (Common::getRequestVar('clientProtocol', 'http', 'string') == 'https') { $protocol = 'https'; } else { $protocol = ProxyHeaders::getProtocolInformation(); } if (!empty($protocol) && !\Piwik\ProxyHttp::isHttps()) { $config->General['assume_secure_protocol'] = '1'; } $config->General['salt'] = Common::generateUniqId(); $config->General['installation_in_progress'] = 1; $config->database = $dbInfos; if (!DbHelper::isDatabaseConnectionUTF8()) { $config->database['charset'] = 'utf8'; } $config->forceSave(); }
/** * @return string returns random 16 chars hex string */ public static function generateUniqueVisitorId() { return substr(Common::generateUniqId(), 0, Tracker::LENGTH_HEX_ID_STRING); }
/** * @param $level * @param $tag * @param $datetime * @param $message * @return string */ private function getMessageFormattedScreen($level, $tag, $datetime, $message) { static $currentRequestKey; if (empty($currentRequestKey)) { $currentRequestKey = substr(Common::generateUniqId(), 0, 5); } if (is_string($message)) { if (!defined('PIWIK_TEST_MODE')) { $message = '[' . $currentRequestKey . '] ' . $message; } $message = $this->formatMessage($level, $tag, $datetime, $message); if (!Common::isPhpCliMode()) { $message = Common::sanitizeInputValue($message); $message = '<pre>' . $message . '</pre>'; } } else { $logger = $this; /** * Triggered when trying to log an object to the screen. Plugins can use * this event to convert objects to strings before they are logged. * * The result of this callback can be HTML so no sanitization is done on the result. * This means **YOU MUST SANITIZE THE MESSAGE YOURSELF** if you use this event. * * **Example** * * public function formatScreenMessage(&$message, $level, $tag, $datetime, $logger) { * if ($message instanceof MyCustomDebugInfo) { * $message = Common::sanitizeInputValue($message->formatForScreen()); * } * } * * @param mixed &$message The object that is being logged. Event handlers should * check if the object is of a certain type and if it is, * set `$message` to the string that should be logged. * @param int $level The log level used with this log entry. * @param string $tag The current plugin that started logging (or if no plugin, * the current class). * @param string $datetime Datetime of the logging call. * @param Log $logger The Log singleton. */ Piwik::postEvent(self::FORMAT_SCREEN_MESSAGE_EVENT, array(&$message, $level, $tag, $datetime, $logger)); } return $message . "\n"; }
protected function insertNewArchiveId() { $numericTable = $this->getTableNumeric(); $idSite = $this->idSite; $this->acquireArchiveTableLock(); $locked = self::PREFIX_SQL_LOCK . Common::generateUniqId(); $date = date("Y-m-d H:i:s"); $insertSql = "INSERT INTO {$numericTable} " . " SELECT ifnull(max(idarchive),0)+1,\n\t\t\t\t\t\t\t\t'" . $locked . "',\n\t\t\t\t\t\t\t\t" . (int) $idSite . ",\n\t\t\t\t\t\t\t\t'" . $date . "',\n\t\t\t\t\t\t\t\t'" . $date . "',\n\t\t\t\t\t\t\t\t0,\n\t\t\t\t\t\t\t\t'" . $date . "',\n\t\t\t\t\t\t\t\t0 " . " FROM {$numericTable} as tb1"; Db::get()->exec($insertSql); $this->releaseArchiveTableLock(); $selectIdSql = "SELECT idarchive FROM {$numericTable} WHERE name = ? LIMIT 1"; $id = Db::get()->fetchOne($selectIdSql, $locked); return $id; }
/** * Write configuration file from session-store */ private function createConfigFile($dbInfos) { $config = Config::getInstance(); // make sure DB sessions are used if the filesystem is NFS if (Filesystem::checkIfFileSystemIsNFS()) { $config->General['session_save_handler'] = 'dbtable'; } if (count($headers = ProxyHeaders::getProxyClientHeaders()) > 0) { $config->General['proxy_client_headers'] = $headers; } if (count($headers = ProxyHeaders::getProxyHostHeaders()) > 0) { $config->General['proxy_host_headers'] = $headers; } if (Common::getRequestVar('clientProtocol', 'http', 'string') == 'https') { $protocol = 'https'; } else { $protocol = ProxyHeaders::getProtocolInformation(); } if (!empty($protocol) && !\Piwik\ProxyHttp::isHttps()) { $config->General['assume_secure_protocol'] = '1'; } $config->General['salt'] = Common::generateUniqId(); $config->General['installation_in_progress'] = 1; $config->database = $dbInfos; if (!DbHelper::isDatabaseConnectionUTF8()) { $config->database['charset'] = 'utf8'; } # Improved Security with IBM Bluemix # With SSL ALWAYS available for all Bluemix apps, let's require all requests # to be made over SSL (https) so that data is NOT sent in the clear. # Non-ssl requests will trigger a # Error: Form security failed. # Please reload the form and check that your cookies are enabled # Reference: http://piwik.org/faq/how-to/faq_91/ # Reference: https://developer.ibm.com/answers/questions/8312/how-do-i-enable-tlsssl-for-my-bluemix-application/ $config->General['assume_secure_protocol'] = 1; $config->General['force_ssl'] = 1; # Setup proxy_client_headers to accurately detect GeoIPs of visiting clients $config->General['proxy_client_headers'] = array("HTTP_X_CLIENT_IP", "HTTP_X_FORWARDED_FOR", "HTTP_X_CLUSTER_CLIENT_IP", "HTTP_CLIENT_IP"); $config->General['proxy_host_headers'] = "HTTP_X_FORWARDED_HOST"; # Implement some default settings that optimize performance $config->General['enabled_periods_UI'] = "day,week,month,year"; $config->General['enabled_periods_API'] = "day,week,month,year"; $config->General['action_category_level_limit'] = 3; $config->General['show_multisites_sparklines'] = 0; $config->General['anonymous_user_enable_use_segments_API'] = 0; $config->General['browser_archiving_disabled_enforce'] = 1; $config->General['enable_create_realtime_segments'] = 0; $config->General['enable_segment_suggested_values'] = 0; $config->General['adding_segment_requires_access'] = "superuser"; $config->General['allow_adding_segments_for_all_websites'] = 0; $config->General['datatable_row_limits'] = "5,10,25,50"; $config->General['enable_browser_archiving_triggering'] = 0; $config->General['multisites_refresh_after_seconds'] = 0; $config->General['enable_delete_old_data_settings_admin'] = 0; $config->General['enable_auto_update'] = 0; $config->Debug['enable_measure_piwik_usage_in_idsite'] = 0; $config->Debug['allow_upgrades_to_beta'] = 0; $config->Tracker['new_visit_api_requires_admin'] = 0; # Let us have this Piwik deploy track itself to get some early data and success :-) # $config->Debug['enable_measure_piwik_usage_in_idsite'] = 1; # Emailing the easy way with IBM Bluemix + the SendGrid Service if (isset($_ENV["REDISHOSTNAME"])) { $config->RedisCache['host'] = $_ENV["REDISHOSTNAME"]; $config->RedisCache['port'] = $_ENV["REDISPORT"]; $config->RedisCache['timeout'] = 0.0; $config->RedisCache['password'] = $_ENV["REDISPASSWORD"]; $config->RedisCache['database'] = 14; $config->ChainedCache['backends'] = array("array", "redis"); } # Let's setup the config files trusted hosts entries to handle # 1...N amount of user-defined IBM Bluemix app routes if (isset($_ENV["APPURIS"])) { foreach ($_ENV["APPURIS"] as $application_uri) { $this->addTrustedHosts("https://" . $application_uri); } } # Emailing the easy way with IBM Bluemix + the SendGrid Service if (isset($_ENV["MAILHOST"])) { $config->mail['transport'] = "smtp"; $config->mail['port'] = 587; $config->mail['type'] = "Plain"; $config->mail['host'] = $_ENV["MAILHOST"]; $config->mail['username'] = $_ENV["MAILUSER"]; $config->mail['password'] = $_ENV["MAILPASSWORD"]; } $config->forceSave(); // re-save the currently viewed language (since we saved the config file, there is now a salt which makes the // existing session cookie invalid) $this->resetLanguageCookie(); }
/** * Installation Step 6: General Set-up (superuser login/password/email and subscriptions) */ function generalSetup() { $this->checkPreviousStepIsValid(__FUNCTION__); $view = new View('@Installation/generalSetup', $this->getInstallationSteps(), __FUNCTION__); $this->skipThisStep(__FUNCTION__); $form = new FormGeneralSetup(); if ($form->validate()) { $superUserInfos = array('login' => $form->getSubmitValue('login'), 'password' => md5($form->getSubmitValue('password')), 'email' => $form->getSubmitValue('email'), 'salt' => Common::generateUniqId()); $this->session->superuser_infos = $superUserInfos; $url = Config::getInstance()->General['api_service_url']; $url .= '/1.0/subscribeNewsletter/'; $params = array('email' => $form->getSubmitValue('email'), 'security' => $form->getSubmitValue('subscribe_newsletter_security'), 'community' => $form->getSubmitValue('subscribe_newsletter_community'), 'url' => Url::getCurrentUrlWithoutQueryString()); if ($params['security'] == '1' || $params['community'] == '1') { if (!isset($params['security'])) { $params['security'] = '0'; } if (!isset($params['community'])) { $params['community'] = '0'; } $url .= '?' . http_build_query($params, '', '&'); try { Http::sendHttpRequest($url, $timeout = 2); } catch (Exception $e) { // e.g., disable_functions = fsockopen; allow_url_open = Off } } $this->redirectToNextStep(__FUNCTION__); } $view->addForm($form); return $view->render(); }
private function getRandomTmpPluginDownloadFilename() { $tmpPluginPath = StaticContainer::get('path.tmp') . '/latest/plugins/'; // we generate a random unique id as filename to prevent any user could possibly download zip directly by // opening $piwikDomain/tmp/latest/plugins/$pluginName.zip in the browser. Instead we make it harder here // and try to make sure to delete file in case of any error. $tmpPluginFolder = Common::generateUniqId(); return $tmpPluginPath . $tmpPluginFolder . '.zip'; }
/** * Locks the archive table to generate a new archive ID. * * We lock to make sure that * if several archiving processes are running at the same time (for different websites and/or periods) * then they will each use a unique archive ID. * * @return int */ public function insertNewArchiveId($numericTable, $idSite, $date) { $this->acquireArchiveTableLock($numericTable); $locked = self::PREFIX_SQL_LOCK . Common::generateUniqId(); $insertSql = "INSERT INTO {$numericTable} " . " SELECT IFNULL( MAX(idarchive), 0 ) + 1,\n '" . $locked . "',\n " . (int) $idSite . ",\n '" . $date . "',\n '" . $date . "',\n 0,\n '" . $date . "',\n 0 " . " FROM {$numericTable} as tb1"; Db::get()->exec($insertSql); $this->releaseArchiveTableLock($numericTable); $selectIdSql = "SELECT idarchive FROM {$numericTable} WHERE name = ? LIMIT 1"; $id = Db::get()->fetchOne($selectIdSql, $locked); return $id; }
/** * Returns either * - "-1" for a known visitor * - at least 16 char identifier in hex @see Common::generateUniqId() * @return int|string */ protected function getVisitorUniqueId() { if ($this->isVisitorKnown()) { return -1; } return Common::generateUniqId(); }
/** * Returns the user's API token. * * If the username/password combination is incorrect an invalid token will be returned. * * @param string $userLogin Login * @param string $md5Password hashed string of the password (using current hash function; MD5-named for historical reasons) * @return string */ public function getTokenAuth($userLogin, $md5Password) { UsersManager::checkPasswordHash($md5Password, Piwik::translate('UsersManager_ExceptionPasswordMD5HashExpected')); $user = $this->model->getUser($userLogin); if (!$this->password->verify($md5Password, $user['password'])) { return md5($userLogin . microtime(true) . Common::generateUniqId()); } if ($this->password->needsRehash($user['password'])) { $this->updateUser($userLogin, $this->password->hash($md5Password)); } return $user['token_auth']; }