Beispiel #1
0
 /**
  * Split shard partitin to new shards. It takes the full table or single partition
  * and split it to multiple partitions according to the number of provided 'parts'.
  * When doing initioa split 'shard' may be ommited however 'fields' shall be provided
  * @method split
  * @static
  * @param {Q_Tree} $config Contains all necessary information for split procedure in the following format:
  * @example
  *	{
  *		"plugin": "PLUGINNAME", // the name of plugin - shall be used by app
  *		"connection": "CONNECTIONNAME", // connection - shall be registered with plugin
  *		"table": "TABLENAME", // the table to shard
  *		"class": "CLASSNAME", // the class which is stored in the table
  *		"fields": {"FIELDNAME": "HASH", "FIELDNAME": "HASH", ...}, // Optional. Used only when starting sharding
  *			"shard": "SHARDNAME" // Optionsl. The shard to split. If no shards defined or SHARDNAME does not exist the script will fail
  *			// "parts" can be either array of connections or object {"SHARDNAME": connection, ...}
  *		"parts": {
  *			"SHARDNAME": {
  *				"prefix": "PREFIX",
  *				"dsn": "DSN",
  *					...
  *			},
  *			"SHARDNAME": {
  *				"prefix": "PREFIX",
  *				"dsn": "DSN",
  *					"username": "******",
  *					"password": "******",
  *					"driver_options": {
  *						"3": 2
  *					}
  *			},
  *			...
  *		}
  *	}
  *
  * @return {boolean} Weather php part of the process completed successfuly
  */
 static function split($config)
 {
     // all input data shall be provided
     // for future extension plugin/connection/table/class are considered unrelated
     if (!($plugin = $config->get('plugin', false))) {
         echo "Plugin name is not defined\n";
         return false;
     }
     // plugin shall be registered!
     if (!Q_Config::get('Q', 'pluginInfo', $plugin, false)) {
         echo "Plugin '{$plugin}' is not registered in the platform\n";
         return false;
     }
     if (!($connection = $config->get('connection', false))) {
         echo "Connection '{$connection}' is not defined\n";
         return false;
     }
     // connection shall exist and be registered with plugin!
     if (!Q_Config::get('Db', 'connections', $connection, false)) {
         echo "Connection '{$connection}' does not exist\n";
         return false;
     }
     if (!in_array($connection, Q_Config::get('Q', 'pluginInfo', $plugin, 'connections', array()))) {
         echo "Connection '{$connection}' is not registered for plugin '{$plugin}'\n";
         return false;
     }
     if (!($class = $config->get('class', false))) {
         echo "Class name is not defined\n";
         return false;
     }
     if (!($table = $config->get('table', false))) {
         echo "Table name is not defined\n";
         return false;
     }
     if (!($shard = $config->get('shard', false)) && Q_Config::get('Db', 'connections', $connection, 'shards', false)) {
         echo "Shard to partition is not defined\n";
         return false;
     }
     if (!($parts = $config->get('parts', false))) {
         echo "New parts are not defined\n";
         return false;
     }
     if ($node = $config->get('node', null)) {
         $nodeInternal = Q_Config::expect('Q', 'nodeInternal');
         $node = array("http://{$nodeInternal['host']}:{$nodeInternal['port']}/Q_Utils/query", $node);
     }
     // now we shall distinguish if table is already sharded or not
     if ($shard === false) {
         if (!($fields = $config->get('fields', false))) {
             echo "To start sharding you shall define 'fields' parameter\n";
             return false;
         }
     }
     // weather provided split config is mapped or not
     $split_mapped = array_keys($parts) !== range(0, count($parts) - 1);
     // set up config for shards if it does not exist yet
     if ($shard === false) {
         $partition = array();
         foreach ($fields as $name => $hash) {
             if (empty($hash)) {
                 $hash = 'md5';
             }
             $part = explode('%', $hash);
             $hash = $part[0];
             $len = isset($part[1]) ? $part[1] : Db_Query::HASH_LEN;
             // "0" has the lowest ascii code for both md5 and normalize
             //	$partition[] = $hash === 'md5' ? str_pad('', $len, "0", STR_PAD_LEFT) : str_pad('', $len, " ", STR_PAD_LEFT);
             $partition[] = str_pad('', $len, "0", STR_PAD_LEFT);
         }
         $shard = join('.', $partition);
         if (Q_Config::get('Db', 'connections', $connection, 'indexes', $table, false)) {
             echo "Shards are not defined but indexes for table '{$table}' are defined in local config\n";
             return false;
         }
         // Let's merge in dummy shards section - shard with name '' is handled as single table
         Q_Config::merge(array('Db' => array('connections' => array($connection => array("shards" => array(), "indexes" => array($table => array("fields" => $fields, "partition" => $split_mapped ? array($shard => '') : array($shard))))))));
         $shard_name = '';
     }
     // get partition information
     if (!($partition = Q_Config::get('Db', 'connections', $connection, 'indexes', $table, 'partition', false))) {
         echo "Upps, cannot get shards partitioning\n";
         return false;
     }
     // weather main config is mapped or not
     // also $points contains the partitioning array without mapping
     $points = ($mapped = array_keys($partition) !== range(0, count($partition) - 1)) ? array_keys($partition) : $partition;
     $i = array_search($shard, $points);
     $next = isset($points[++$i]) ? $points[$i] : null;
     $fields = Q_Config::expect('Db', 'connections', $connection, 'indexes', $table, 'fields');
     // now $shard and $next contain boundaries for data to split
     // $points contain partitioning array without mapping - array
     // $parts contains split parts (shards) definition - array or object ($split_mapped)
     // $partition contains current partitioning - array or object ($mapped)
     // $fields contains field names and hashes
     // time to calculate new split point(s)
     if (!isset($shard_name)) {
         $shard_name = $mapped ? $partition[$shard] : $shard;
     }
     $shard_db = $class::db();
     $pdo = $shard_db->reallyConnect($shard_name);
     $shard_table = $class::table();
     $shard_table = str_replace('{$dbname}', $shard_db->dbname, $shard_table);
     $shard_table = str_replace('{$prefix}', $shard_db->prefix, $shard_table);
     // verify if current shard is updated to latest version
     $current_version = $shard_db->select('version', "{$shard_db->prefix}Q_plugin")->where(array("plugin" => $plugin))->fetchAll(PDO::FETCH_ASSOC);
     if (!empty($current_version)) {
         $current_version = $current_version[0]['version'];
         $version = Q_Config::get('Q', "pluginInfo", $plugin, 'version', null);
         if (Q::compareVersion($current_version, $version) < 0) {
             echo "Please, update plugin '{$plugin}' to version '{$version}' (currently {$current_version})\n";
             return false;
         }
     } else {
         echo "Cannot get installed version of plugin '{$plugin}'\n";
         return false;
     }
     // We'll limit search with shard boundaries using latin1 string comparison
     $lower = join(explode('.', $shard));
     $upper = isset($next) ? join(explode('.', $next)) : null;
     $normalize = false;
     $where = $group = $order = array();
     foreach (array_keys($fields) as $i => $field) {
         $hash = !empty($fields[$field]) ? $fields[$field] : 'md5';
         $part = explode('%', $hash);
         $normalize = $normalize || ($hash = strtoupper($part[0])) === 'NORMALIZE';
         $len = isset($part[1]) ? $part[1] : Db_Query::HASH_LEN;
         $group[] = $field;
         $order[] = "CAST({$hash}({$field}) AS CHAR({$len}))";
     }
     // if any field uses 'normalize' hash
     // the original shard shall have MySQL NORMALIZE() function defined
     // MySQL version of NORMALIZE handles only 255 chars and does not add md5 hash
     // (see Db_Utils::normalize)
     if ($normalize) {
         try {
             $pdo->exec("DROP FUNCTION IF EXISTS NORMALIZE;");
             $pdo->exec("CREATE FUNCTION NORMALIZE(s CHAR(255))\n\t\t\t\t\t\tRETURNS CHAR(255) DETERMINISTIC\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t    \tDECLARE res CHAR(255) DEFAULT '';\n\t\t\t\t\t  \t\tDECLARE t CHAR(1);\n\t\t\t\t\t    \tWHILE LENGTH(s) > 0 DO\n\t\t\t\t\t        \tSET t = LOWER(LEFT(s, 1));\n\t\t\t\t\t    \t    SET s = SUBSTRING(s FROM 2);\n\t\t\t\t\t        \tIF t REGEXP '[^A-Za-z0-9]' THEN\n\t\t\t\t\t            \tSET t = '_';\n\t\t\t\t\t        \tEND IF;\n\t\t\t\t\t        \tSET res = CONCAT(res, t);\n\t\t\t\t\t    \tEND WHILE;\n\t\t\t\t\t    \tRETURN res;\n\t\t\t\t\t\tEND");
         } catch (Exception $e) {
             //echo "ERROR: {$e->getMessage()}\n";
             echo "Please, make sure that db user for shard '{$shard_name}' has 'CREATE ROUTINE' permission\n";
             return false;
         }
     }
     $order = join(', ', $order);
     $group = join(', ', $group);
     $where = "(STRCMP(CONCAT({$order}), '{$lower}') >= 0)" . (isset($upper) ? " AND (STRCMP(CONCAT({$order}), '{$upper}') < 0)" : "");
     $count = reset($pdo->query("SELECT COUNT(*) FROM {$shard_table} WHERE {$where}")->fetchAll(PDO::FETCH_NUM));
     if (empty($count)) {
         echo "Failed to connect to shard '{$shard_name}'\n";
         return false;
     }
     $count = reset($count);
     if ($count == 0) {
         echo "Cannot split empty shard!\n";
         return false;
     }
     // if only one new shard provided script will copy data and cnange config
     if (($num_shards = count($parts)) < 1) {
         echo "Please, provide at least one new shard";
         return false;
     }
     $break = round($count / $num_shards);
     // if split config is not mapped and current config is mapped we shall convert split
     //  config to mapped
     $new_partition = $mapped || $split_mapped ? array($shard => $split_mapped ? reset(array_keys($parts)) : $shard) : array($shard);
     $new_shards = array($split_mapped ? reset(array_keys($parts)) : $shard => reset($parts));
     $i = 0;
     foreach (array_slice($parts, 1) as $name => $dsn) {
         $offset = $break * ++$i;
         $split = reset($pdo->query("SELECT {$group} FROM {$shard_table} WHERE {$where} ORDER BY {$order} LIMIT {$offset}, 1")->fetchAll(PDO::FETCH_ASSOC));
         foreach ($fields as $field => $hash) {
             $split[$field] = Db_Query::hashed($split[$field], $hash);
         }
         $split = join('.', $split);
         if ($mapped || $split_mapped) {
             $new_partition[$split] = $split_mapped ? $name : $split;
         } else {
             $new_partition[] = $split;
         }
         $new_shards[$new_name = $split_mapped ? $name : $split] = $dsn;
         if (Q_Config::get('Db', 'connections', $connection, 'shards', $new_name, false)) {
             echo "WARNING!!! Shard already exists: '{$new_name}'\n";
         }
     }
     Q_Config::merge(array('Db' => array('connections' => array($connection => array("shards" => $new_shards)))));
     // if split config is mapped and current config is not we shall convert app config to mapped
     if ($split_mapped && !$mapped) {
         $partition = array();
         foreach ($points as $point) {
             $partition[$point] = $point;
         }
         Q_Config::set('Db', 'connections', $connection, 'indexes', $table, 'partition', $partition);
         $mapped = true;
     }
     // TODO: verify if new shards sizes are approx. equal
     // Verify versions of existing shards and
     // Install pligin schema to new shards
     Q_Plugin::installSchema(Q_PLUGINS_DIR . DS . $plugin, $plugin, 'plugin', $connection, array('sql' => array($connection => array('enabled' => true))));
     // make sure 'upcoming' config is loaded
     $configFiles = Q_Config::get('Q', 'configFiles', array());
     // 'local/Q/bootstrap.json' should be loaded already but we'll better check
     if (!in_array('Q/config/bootstrap.json', $configFiles)) {
         echo "Config file 'Q/config/bootstrap.json' shall be loaded via 'Q/configFiles key'\non every Q server - check 'platform/config/Q.json'\n";
         return false;
     }
     $upcoming_file = Q_Config::get('Q', 'internal', 'sharding', 'upcoming', 'Db/config/upcoming.json');
     //if (!unlink ($upcoming_file)) {
     //	echo "Please, manually remove file '$upcoming_file' and start this script again.\n";
     //	return false;
     //}
     if (!in_array($upcoming_file, $configFiles)) {
         // add upcoming.json to config
         if (!Q_Config::setOnServer('Q/config/bootstrap.json', array('Q' => array('configFiles' => array($upcoming_file))))) {
             echo "Failed to update 'local/Q/bootstrap.json'\n";
             return false;
         }
     }
     // Now after some short time all workers (php and node) will be ready for splitting
     // We'll let node server to wait necessary amount of time.
     $res = Q_Utils::queryInternal('Db/Shards', array('Q/method' => 'split', 'shard' => $shard_name, 'shards' => Q::json_encode($new_shards), 'part' => $shard, 'table' => $table, 'dbTable' => $shard_table, 'class' => $class, 'plugin' => $plugin, 'connection' => $connection, 'where' => $where, 'parts' => Q::json_encode(array('partition' => $new_partition, 'fields' => $fields))), $node);
     if ($res) {
         echo "Split process for shard '{$shard_name}' ({$shard}) has started\nPlease, monitor node.js console for important messages and process status\n";
         return true;
     }
     echo "Failed to start split process at node server\n";
     return false;
 }
Beispiel #2
0
 /**
  * @method checkRequirementsApp
  * @static
  * @throws {Q_Exception_AppRequirement}
  * @throws {Q_Exception_AppRequirementVersion}
  */
 static function checkRequirementsApp()
 {
     $title = Q_Config::expect('Q', 'app');
     $required = Q_Config::get('Q', 'appInfo', 'requires', array());
     $local = Q_Config::get('Q', 'pluginLocal', array());
     $local['Q'] = array('version' => Q_VERSION, 'compatible' => Q_VERSION);
     foreach ($required as $required_plugin => $required_version) {
         if (!isset($local[$required_plugin])) {
             throw new Q_Exception_AppRequirement(array('plugin' => $required_plugin, 'version' => $required_version, 'by' => $title));
         }
         $installed_version = isset($local[$required_plugin]['version']) ? $local[$required_plugin]['version'] : 0;
         $compatible_version = isset($local[$required_plugin]['compatible']) ? $local[$required_plugin]['compatible'] : 0;
         if (Q::compareVersion($installed_version, $required_version) < 0) {
             throw new Q_Exception_AppRequirementVersion(array('plugin' => $required_plugin, 'version' => $required_version, 'by' => $title, 'installed' => $installed_version, 'compatible' => $compatible_version));
         }
         if (Q::compareVersion($compatible_version, $required_version) > 0) {
             throw new Q_Exception_AppRequirementVersion(array('plugin' => $required_plugin, 'version' => $required_version, 'by' => $title, 'installed' => $installed_version, 'compatible' => $compatible_version));
         }
     }
 }
Beispiel #3
0
 /**
  * @method installPlugin
  * @static
  * @param {string} $plugin_name
  * @param {array} $options
  * @throws {Exception}
  */
 static function installPlugin($plugin_name, $options)
 {
     set_time_limit(Q_Config::expect('Q', 'install', 'timeLimit'));
     // Connect Qbix platform if it's not already connected
     self::prepare();
     $app_dir = APP_DIR;
     $plugin_dir = Q_PLUGINS_DIR . DS . $plugin_name;
     $app_web_plugins_dir = APP_WEB_DIR . DS . 'plugins';
     echo "Installing plugin '{$plugin_name}' into '{$app_dir}'" . PHP_EOL;
     // Do we even have such a plugin?
     if (!is_dir($plugin_dir)) {
         throw new Exception("Plugin '{$plugin_name}' not found in " . Q_PLUGINS_DIR);
     }
     // Ensure that the plugin has config.json
     if (!file_exists($plugin_conf_file = $plugin_dir . DS . 'config' . DS . 'plugin.json')) {
         throw new Exception("Could not load plugin's config. Check {$plugin_conf_file}");
     }
     $files_dir = $plugin_dir . DS . 'files';
     $app_plugins_file = APP_LOCAL_DIR . DS . 'plugins.json';
     // Check access to $app_web_plugins_dir
     if (!file_exists($app_web_plugins_dir)) {
         if (!@mkdir($app_web_plugins_dir, 0755, true)) {
             throw new Exception("Could not create {$app_web_plugins_dir}");
         }
     }
     if (!is_dir($app_web_plugins_dir)) {
         throw new Exception("{$app_web_plugins_dir} exists, but is not a directory");
     } elseif (!is_writable($app_web_plugins_dir)) {
         throw new Exception("Can not write to {$app_web_plugins_dir}");
     }
     // Check access to $app_plugins_file
     if (file_exists($app_plugins_file) && !is_writable($app_plugins_file)) {
         throw new Exception("Can not write to {$app_plugins_file}");
     } elseif (!file_exists($app_plugins_file) && !is_writable(dirname($app_plugins_file))) {
         throw new Exception("Can not write to " . dirname($app_plugins_file));
     }
     // Check access to $files_dir
     if (!file_exists($files_dir)) {
         if (!@mkdir($files_dir, $options['dirmode'], true)) {
             throw new Exception("Could not create {$files_dir}");
         }
     }
     // Do we now have plugin's config?
     if (Q_Config::get('Q', 'pluginInfo', $plugin_name, 'version', null) == null) {
         throw new Exception("Could not identify plugin version. Check {$plugin_conf_file}");
     }
     $plugin_conf = Q_Config::get('Q', 'pluginInfo', $plugin_name, null);
     $plugin_version = $plugin_conf['version'];
     if (file_exists($app_plugins_file)) {
         Q_Config::load($app_plugins_file, true);
     }
     //  Do we already have this plugin installed for this app?
     // Check requirements for plugin (will throw exceptions if they aren't met)
     if (!isset($options['noreq']) || !$options['noreq']) {
         echo "Checking requirements" . PHP_EOL;
         Q_Bootstrap::checkRequirements(array($plugin_name));
     }
     //  Checking LOCAL plugin version in plugins.json file
     if (($version_installed = Q_Config::get('Q', 'pluginLocal', $plugin_name, 'version', null)) != null) {
         //We have this plugin installed
         echo "Plugin '{$plugin_name}' (version: {$version_installed}) is already installed" . PHP_EOL;
         if (Q::compareVersion($version_installed, $plugin_version) < 0) {
             echo "Upgrading '{$plugin_name}' to version: {$plugin_version}" . PHP_EOL;
         }
     }
     // Check and fix permissions
     self::checkPermissions($files_dir, $options);
     if (isset($plugin_conf['permissions'])) {
         foreach ($plugin_conf['permissions'] as $perm) {
             self::checkPermissions($files_dir . DS . $perm, $options);
         }
     }
     // Symbolic links
     echo 'Creating symbolic links' . PHP_EOL;
     Q_Utils::symlink($plugin_dir . DS . 'web', $app_web_plugins_dir . DS . $plugin_name);
     //  Checking if schema update is requested and updating database version
     $connections = Q_Config::get('Q', 'pluginInfo', $plugin_name, 'connections', array());
     foreach ($connections as $connection) {
         self::installSchema(Q_PLUGINS_DIR . DS . $plugin_name, $plugin_name, 'plugin', $connection, $options);
     }
     // Push plugin name into Q/plugins array
     if (!in_array($plugin_name, $current_plugins = Q_Config::get('Q', 'plugins', array()))) {
         $current_plugins[] = $plugin_name;
         Q_Config::set('Q', 'plugins', $current_plugins);
         //TODO: When do we save Q/plugins to disk?
     }
     // Save info about plugin
     echo 'Registering plugin' . PHP_EOL;
     Q_Config::set('Q', 'pluginLocal', $plugin_name, $plugin_conf);
     Q_Config::save($app_plugins_file, array('Q', 'pluginLocal'));
     echo Q_Utils::colored("Plugin '{$plugin_name}' successfully installed" . PHP_EOL, 'green');
 }