protected function _partConfig(Db_Query $query, Db_Query_Part $part)
{
$cfg = array();
$fields = $part->getFields();
$objectConfig = Db_Object_Config::getInstance($part->getObject());
$mainPanel = new stdClass();
$mainPanel->xtype = 'panel';
$mainPanel->border = false;
ksort($fields);
foreach ($fields as $name => $config) {
$obj = new stdClass();
$obj->xtype = 'reportfield';
$obj->valueSelected = $config['select'];
$obj->valueTitle = $config['title'];
$obj->valueAlias = $config['alias'];
$obj->valueField = $name;
$obj->valueIsLink = false;
$obj->valueSelectSub = $config['selectSub'];
$obj->valueObject = $part->getObject();
$obj->valuePartId = $part->getId();
$obj->valueSubObject = '';
$obj->valueSubObjectTtile = '';
if ($config['isLink'] && !$objectConfig->isDictionaryLink($name)) {
$obj->valueIsLink = true;
$child = $query->findChild($part->getId(), $name);
if ($child !== false) {
$linked = $child->getObject();
} else {
$linked = $objectConfig->getLinkedObject($name);
}
if ($linked) {
$obj->valueSubObject = $linked;
$obj->valueSubObjectTtile = Db_Object_Config::getInstance($linked)->get('title');
}
}
$cfg[] = $obj;
}
$mainPanel->items = $cfg;
return array('items' => $mainPanel, 'objectcfg' => array('join' => $part->joinType, 'title' => $objectConfig->get('title'), 'object' => $part->getObject(), 'childField' => $part->getChildField()));
}
/**
* Split shard partitin to new shards. It takes the full table or single partition
* and split it to multiple partitions according to the number of provided 'parts'.
* When doing initioa split 'shard' may be ommited however 'fields' shall be provided
* @method split
* @static
* @param {Q_Tree} $config Contains all necessary information for split procedure in the following format:
* @example
* {
* "plugin": "PLUGINNAME", // the name of plugin - shall be used by app
* "connection": "CONNECTIONNAME", // connection - shall be registered with plugin
* "table": "TABLENAME", // the table to shard
* "class": "CLASSNAME", // the class which is stored in the table
* "fields": {"FIELDNAME": "HASH", "FIELDNAME": "HASH", ...}, // Optional. Used only when starting sharding
* "shard": "SHARDNAME" // Optionsl. The shard to split. If no shards defined or SHARDNAME does not exist the script will fail
* // "parts" can be either array of connections or object {"SHARDNAME": connection, ...}
* "parts": {
* "SHARDNAME": {
* "prefix": "PREFIX",
* "dsn": "DSN",
* ...
* },
* "SHARDNAME": {
* "prefix": "PREFIX",
* "dsn": "DSN",
* "username": "******",
* "password": "******",
* "driver_options": {
* "3": 2
* }
* },
* ...
* }
* }
*
* @return {boolean} Weather php part of the process completed successfuly
*/
static function split($config)
{
// all input data shall be provided
// for future extension plugin/connection/table/class are considered unrelated
if (!($plugin = $config->get('plugin', false))) {
echo "Plugin name is not defined\n";
return false;
}
// plugin shall be registered!
if (!Q_Config::get('Q', 'pluginInfo', $plugin, false)) {
echo "Plugin '{$plugin}' is not registered in the platform\n";
return false;
}
if (!($connection = $config->get('connection', false))) {
echo "Connection '{$connection}' is not defined\n";
return false;
}
// connection shall exist and be registered with plugin!
if (!Q_Config::get('Db', 'connections', $connection, false)) {
echo "Connection '{$connection}' does not exist\n";
return false;
}
if (!in_array($connection, Q_Config::get('Q', 'pluginInfo', $plugin, 'connections', array()))) {
echo "Connection '{$connection}' is not registered for plugin '{$plugin}'\n";
return false;
}
if (!($class = $config->get('class', false))) {
echo "Class name is not defined\n";
return false;
}
if (!($table = $config->get('table', false))) {
echo "Table name is not defined\n";
return false;
}
if (!($shard = $config->get('shard', false)) && Q_Config::get('Db', 'connections', $connection, 'shards', false)) {
echo "Shard to partition is not defined\n";
return false;
}
if (!($parts = $config->get('parts', false))) {
echo "New parts are not defined\n";
return false;
}
if ($node = $config->get('node', null)) {
$nodeInternal = Q_Config::expect('Q', 'nodeInternal');
$node = array("http://{$nodeInternal['host']}:{$nodeInternal['port']}/Q_Utils/query", $node);
}
// now we shall distinguish if table is already sharded or not
if ($shard === false) {
if (!($fields = $config->get('fields', false))) {
echo "To start sharding you shall define 'fields' parameter\n";
return false;
}
}
// weather provided split config is mapped or not
$split_mapped = array_keys($parts) !== range(0, count($parts) - 1);
// set up config for shards if it does not exist yet
if ($shard === false) {
$partition = array();
foreach ($fields as $name => $hash) {
if (empty($hash)) {
$hash = 'md5';
}
$part = explode('%', $hash);
$hash = $part[0];
$len = isset($part[1]) ? $part[1] : Db_Query::HASH_LEN;
// "0" has the lowest ascii code for both md5 and normalize
// $partition[] = $hash === 'md5' ? str_pad('', $len, "0", STR_PAD_LEFT) : str_pad('', $len, " ", STR_PAD_LEFT);
$partition[] = str_pad('', $len, "0", STR_PAD_LEFT);
}
$shard = join('.', $partition);
if (Q_Config::get('Db', 'connections', $connection, 'indexes', $table, false)) {
echo "Shards are not defined but indexes for table '{$table}' are defined in local config\n";
return false;
}
// Let's merge in dummy shards section - shard with name '' is handled as single table
Q_Config::merge(array('Db' => array('connections' => array($connection => array("shards" => array(), "indexes" => array($table => array("fields" => $fields, "partition" => $split_mapped ? array($shard => '') : array($shard))))))));
$shard_name = '';
}
// get partition information
if (!($partition = Q_Config::get('Db', 'connections', $connection, 'indexes', $table, 'partition', false))) {
echo "Upps, cannot get shards partitioning\n";
return false;
}
// weather main config is mapped or not
// also $points contains the partitioning array without mapping
$points = ($mapped = array_keys($partition) !== range(0, count($partition) - 1)) ? array_keys($partition) : $partition;
$i = array_search($shard, $points);
$next = isset($points[++$i]) ? $points[$i] : null;
$fields = Q_Config::expect('Db', 'connections', $connection, 'indexes', $table, 'fields');
// now $shard and $next contain boundaries for data to split
// $points contain partitioning array without mapping - array
// $parts contains split parts (shards) definition - array or object ($split_mapped)
// $partition contains current partitioning - array or object ($mapped)
// $fields contains field names and hashes
// time to calculate new split point(s)
if (!isset($shard_name)) {
$shard_name = $mapped ? $partition[$shard] : $shard;
}
$shard_db = $class::db();
$pdo = $shard_db->reallyConnect($shard_name);
$shard_table = $class::table();
$shard_table = str_replace('{$dbname}', $shard_db->dbname, $shard_table);
$shard_table = str_replace('{$prefix}', $shard_db->prefix, $shard_table);
// verify if current shard is updated to latest version
$current_version = $shard_db->select('version', "{$shard_db->prefix}Q_plugin")->where(array("plugin" => $plugin))->fetchAll(PDO::FETCH_ASSOC);
if (!empty($current_version)) {
$current_version = $current_version[0]['version'];
$version = Q_Config::get('Q', "pluginInfo", $plugin, 'version', null);
if (Q::compareVersion($current_version, $version) < 0) {
echo "Please, update plugin '{$plugin}' to version '{$version}' (currently {$current_version})\n";
return false;
}
} else {
echo "Cannot get installed version of plugin '{$plugin}'\n";
return false;
}
// We'll limit search with shard boundaries using latin1 string comparison
$lower = join(explode('.', $shard));
$upper = isset($next) ? join(explode('.', $next)) : null;
$normalize = false;
$where = $group = $order = array();
foreach (array_keys($fields) as $i => $field) {
$hash = !empty($fields[$field]) ? $fields[$field] : 'md5';
$part = explode('%', $hash);
$normalize = $normalize || ($hash = strtoupper($part[0])) === 'NORMALIZE';
$len = isset($part[1]) ? $part[1] : Db_Query::HASH_LEN;
$group[] = $field;
$order[] = "CAST({$hash}({$field}) AS CHAR({$len}))";
}
// if any field uses 'normalize' hash
// the original shard shall have MySQL NORMALIZE() function defined
// MySQL version of NORMALIZE handles only 255 chars and does not add md5 hash
// (see Db_Utils::normalize)
if ($normalize) {
try {
$pdo->exec("DROP FUNCTION IF EXISTS NORMALIZE;");
$pdo->exec("CREATE FUNCTION NORMALIZE(s CHAR(255))\n\t\t\t\t\t\tRETURNS CHAR(255) DETERMINISTIC\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t \tDECLARE res CHAR(255) DEFAULT '';\n\t\t\t\t\t \t\tDECLARE t CHAR(1);\n\t\t\t\t\t \tWHILE LENGTH(s) > 0 DO\n\t\t\t\t\t \tSET t = LOWER(LEFT(s, 1));\n\t\t\t\t\t \t SET s = SUBSTRING(s FROM 2);\n\t\t\t\t\t \tIF t REGEXP '[^A-Za-z0-9]' THEN\n\t\t\t\t\t \tSET t = '_';\n\t\t\t\t\t \tEND IF;\n\t\t\t\t\t \tSET res = CONCAT(res, t);\n\t\t\t\t\t \tEND WHILE;\n\t\t\t\t\t \tRETURN res;\n\t\t\t\t\t\tEND");
} catch (Exception $e) {
//echo "ERROR: {$e->getMessage()}\n";
echo "Please, make sure that db user for shard '{$shard_name}' has 'CREATE ROUTINE' permission\n";
return false;
}
}
$order = join(', ', $order);
$group = join(', ', $group);
$where = "(STRCMP(CONCAT({$order}), '{$lower}') >= 0)" . (isset($upper) ? " AND (STRCMP(CONCAT({$order}), '{$upper}') < 0)" : "");
$count = reset($pdo->query("SELECT COUNT(*) FROM {$shard_table} WHERE {$where}")->fetchAll(PDO::FETCH_NUM));
if (empty($count)) {
echo "Failed to connect to shard '{$shard_name}'\n";
return false;
}
$count = reset($count);
if ($count == 0) {
echo "Cannot split empty shard!\n";
return false;
}
// if only one new shard provided script will copy data and cnange config
if (($num_shards = count($parts)) < 1) {
echo "Please, provide at least one new shard";
return false;
}
$break = round($count / $num_shards);
// if split config is not mapped and current config is mapped we shall convert split
// config to mapped
$new_partition = $mapped || $split_mapped ? array($shard => $split_mapped ? reset(array_keys($parts)) : $shard) : array($shard);
$new_shards = array($split_mapped ? reset(array_keys($parts)) : $shard => reset($parts));
$i = 0;
foreach (array_slice($parts, 1) as $name => $dsn) {
$offset = $break * ++$i;
$split = reset($pdo->query("SELECT {$group} FROM {$shard_table} WHERE {$where} ORDER BY {$order} LIMIT {$offset}, 1")->fetchAll(PDO::FETCH_ASSOC));
foreach ($fields as $field => $hash) {
$split[$field] = Db_Query::hashed($split[$field], $hash);
}
$split = join('.', $split);
if ($mapped || $split_mapped) {
$new_partition[$split] = $split_mapped ? $name : $split;
} else {
$new_partition[] = $split;
}
$new_shards[$new_name = $split_mapped ? $name : $split] = $dsn;
if (Q_Config::get('Db', 'connections', $connection, 'shards', $new_name, false)) {
echo "WARNING!!! Shard already exists: '{$new_name}'\n";
}
}
Q_Config::merge(array('Db' => array('connections' => array($connection => array("shards" => $new_shards)))));
// if split config is mapped and current config is not we shall convert app config to mapped
if ($split_mapped && !$mapped) {
$partition = array();
foreach ($points as $point) {
$partition[$point] = $point;
}
Q_Config::set('Db', 'connections', $connection, 'indexes', $table, 'partition', $partition);
$mapped = true;
}
// TODO: verify if new shards sizes are approx. equal
// Verify versions of existing shards and
// Install pligin schema to new shards
Q_Plugin::installSchema(Q_PLUGINS_DIR . DS . $plugin, $plugin, 'plugin', $connection, array('sql' => array($connection => array('enabled' => true))));
// make sure 'upcoming' config is loaded
$configFiles = Q_Config::get('Q', 'configFiles', array());
// 'local/Q/bootstrap.json' should be loaded already but we'll better check
if (!in_array('Q/config/bootstrap.json', $configFiles)) {
echo "Config file 'Q/config/bootstrap.json' shall be loaded via 'Q/configFiles key'\non every Q server - check 'platform/config/Q.json'\n";
return false;
}
$upcoming_file = Q_Config::get('Q', 'internal', 'sharding', 'upcoming', 'Db/config/upcoming.json');
//if (!unlink ($upcoming_file)) {
// echo "Please, manually remove file '$upcoming_file' and start this script again.\n";
// return false;
//}
if (!in_array($upcoming_file, $configFiles)) {
// add upcoming.json to config
if (!Q_Config::setOnServer('Q/config/bootstrap.json', array('Q' => array('configFiles' => array($upcoming_file))))) {
echo "Failed to update 'local/Q/bootstrap.json'\n";
return false;
}
}
// Now after some short time all workers (php and node) will be ready for splitting
// We'll let node server to wait necessary amount of time.
$res = Q_Utils::queryInternal('Db/Shards', array('Q/method' => 'split', 'shard' => $shard_name, 'shards' => Q::json_encode($new_shards), 'part' => $shard, 'table' => $table, 'dbTable' => $shard_table, 'class' => $class, 'plugin' => $plugin, 'connection' => $connection, 'where' => $where, 'parts' => Q::json_encode(array('partition' => $new_partition, 'fields' => $fields))), $node);
if ($res) {
echo "Split process for shard '{$shard_name}' ({$shard}) has started\nPlease, monitor node.js console for important messages and process status\n";
return true;
}
echo "Failed to start split process at node server\n";
return false;
}
/**
* Analyzes the query's criteria and decides where to execute the query.
* Here is sample shards config:
*
* **NOTE:** *"fields" shall be an object with keys as fields names and values containing hash definition
* in the format "type%length" where type is one of 'md5' or 'normalize' and length is hash length
* hash definition can be empty string or false. In such case 'md5%7' is used*
*
* **NOTE:** *"partition" can be an array. In such case shards shall be named after partition points*
*
*
* "Streams": {
* "prefix": "streams_",
* "dsn": "mysql:host=127.0.0.1;dbname=DBNAME",
* "username": "******",
* "password": "******",
* "driver_options": {
* "3": 2
* },
* "shards": {
* "alpha": {
* "prefix": "alpha_",
* "dsn": "mysql:host=127.0.0.1;dbname=SHARDDBNAME",
* "username": "******",
* "password": "******",
* "driver_options": {
* "3": 2
* }
* },
* "betta": {
* "prefix": "betta_",
* "dsn": "mysql:host=127.0.0.1;dbname=SHARDDBNAME",
* "username": "******",
* "password": "******",
* "driver_options": {
* "3": 2
* }
* },
* "gamma": {
* "prefix": "gamma_",
* "dsn": "mysql:host=127.0.0.1;dbname=SHARDDBNAME",
* "username": "******",
* "password": "******",
* "driver_options": {
* "3": 2
* }
* },
* "delta": {
* "prefix": "delta_",
* "dsn": "mysql:host=127.0.0.1;dbname=SHARDDBNAME",
* "username": "******",
* "password": "******",
* "driver_options": {
* "3": 2
* }
* }
* },
* "indexes": {
* "Stream": {
* "fields": {"publisherId": "md5", "name": "normalize"},
* "partition": {
* "0000000. ": "alpha",
* "0000000.sample_": "betta",
* "4000000. ": "gamma",
* "4000000.sample_": "delta",
* "8000000. ": "alpha",
* "8000000.sample_": "betta",
* "c000000. ": "gamma",
* "c000000.sample_": "delta"
* }
* }
* }
* }
*
* @method shard
* @param {array} [$upcoming=null] Temporary config to use in sharding. Used during shard split process only
* @param {array} [$criteria=null] Rarely used unless testing what shards the query would be executed on. Overrides the sharding criteria for the query.
* @return {array} Returns an array of ($shardName => $query) pairs, where $shardName
* can be the name of a shard, '' for just the main shard, or "*" to have the query run on all the shards.
*/
function shard($upcoming = null, $criteria = null)
{
if (isset($criteria)) {
$this->criteria = $criteria;
}
$index = $this->shardIndex();
if (!$index) {
return array("" => $this);
}
if (empty($this->criteria)) {
return array("*" => $this);
}
if (empty($index['fields'])) {
throw new Exception("Db_Query: index for {$this->className} should have at least one field");
}
if (!isset($index['partition'])) {
return array("" => $this);
}
$hashed = array();
$fields = array_keys($index['fields']);
foreach ($fields as $i => $field) {
if (!isset($this->criteria[$field])) {
// not enough information to target the query
return array("*" => $this);
}
$value = $this->criteria[$field];
$hash = !empty($index['fields'][$field]) ? $index['fields'][$field] : 'md5';
$parts = explode('%', $hash);
$hash = $parts[0];
$len = isset($parts[1]) ? $parts[1] : self::HASH_LEN;
if (is_array($value)) {
$arr = array();
foreach ($value as $v) {
$arr[] = self::applyHash($v, $hash, $len);
}
$hashed[$i] = $arr;
} else {
if ($value instanceof Db_Range) {
if ($hash !== 'normalize') {
throw new Exception("Db_Query: ranges don't work with {$hash} hash");
}
$hashed_min = self::applyHash($value->min, $hash, $len);
$hashed_max = self::applyHash($value->max, $hash, $len);
$hashed[$i] = new Db_Range($hashed_min, $value->includeMin, $value->includeMax, $hashed_max);
} else {
$hashed[$i] = self::applyHash($value, $hash, $len);
}
}
}
if (array_keys($index['partition']) === range(0, count($index['partition']) - 1)) {
// $index['partition'] is simple array, name the shards after the partition points
self::$mapping = array_combine($index['partition'], $index['partition']);
} else {
self::$mapping = $index['partition'];
}
return $this->shard_internal($index, $hashed);
}
/**
* set default params
*
* @param array $default
*/
public static function setDefault(array $default)
{
self::$_default = array_merge(self::$_default, $default);
}
