public function execute()
{
if (!$this->isReady('thedate', 'priority')) {
return self::IGNO;
}
$date1 = date('Ymd', strtotime(Setting::get('last_date')));
$date2 = date('Ymd', strtotime($this->option('thedate')));
if ($date1 >= $date2) {
return self::IGNO;
}
$mysql = \Myfox\Lib\Mysql::instance('default');
$count = (int) $mysql->getOne($mysql->query(sprintf('SELECT COUNT(*) FROM %stask_queque WHERE autokid < %u AND priority <= %u ' . "AND task_flag NOT IN (%d,%d) AND task_type='import'", $mysql->option('prefix'), $this->id, $this->option('priority'), \Myfox\App\Queque::FLAG_IGNO, \Myfox\App\Queque::FLAG_DONE)));
if ($count > 0) {
$this->setError(sprintf('Waiting for %d import task(s)', $count));
return self::FAIL;
}
if (!Router::flush()) {
$this->setError('flush route failed');
return self::FAIL;
}
Setting::set('last_date', $date2, '', '集群数据最新日期');
// TODO:
// CALL nodefox to reload route info info cache
return self::SUCC;
}
public function test_should_setting_set_without_comma_works_fine()
{
Setting::init(0);
$this->assertEquals(1, Setting::set('key2', 1, 'unittest1'));
$this->assertEquals(1, Setting::get('key2', 'unittest1'));
$this->assertEquals(2, Setting::set('key2', 'IF(cfgvalue + 0 > 0, 2, 3)', 'unittest1', false));
$this->assertEquals(2, Setting::get('key2', 'unittest1'));
}
public function test_should_import_index_works_fine()
{
$controller = new \Myfox\App\Control\Import();
\Myfox\App\Setting::set('last_date', '20121031');
ob_start();
$controller->execute('index', array());
$output = ob_get_contents();
$this->assertContains('[0] OK', $output);
$this->assertContains('"last_date":"20121031"', $output);
@ob_clean();
}
public function test_should_import_index_works_fine()
{
$controller = new \Myfox\App\Control\Import();
\Myfox\App\Setting::set('myfox_last_date', '20121031');
ob_start();
$controller->execute('index', array());
$output = ob_get_contents();
$this->assertContains('[0] OK', $output);
$this->assertContains('"last_date":"20121031"', $output);
@ob_clean();
$controller->execute('lastdate', array());
$output = ob_get_contents();
$this->assertEquals($output, json_encode(array('group_name' => '__global__', 'last_date' => '20121031')));
@ob_clean();
}
public function test_should_mirror_table_router_set_and_get_works_fine()
{
try {
Router::set('i am not exists');
$this->assertTrue(false);
} catch (\Exception $e) {
$this->assertTrue($e instanceof \Myfox\Lib\Exception);
$this->assertContains('Undefined table named as "i am not exists"', $e->getMessage());
}
$mirror = \Myfox\App\Model\Table::instance('mirror');
foreach (array(0, 1, 2, 0) as $key => $id) {
Setting::set('last_assign_host', 0);
$this->assertEquals(array('' => array(array('rows' => 1300, 'hosts' => '3,4,1,2', 'table' => 'mirror_0.t_' . $mirror->get('autokid') . '_' . $id))), Router::set('mirror', array(array('count' => 1300))));
if ($key == 0) {
$this->assertEquals(array(), Router::get('mirror'));
}
}
$this->assertEquals(1, Setting::get('last_assign_host'));
$this->assertEquals(4, Setting::get('table_route_count', 'mirror'));
$this->assertEquals(0, (int) Setting::get('table_real_count', 'mirror'));
$where = Router::instance('mirror')->where(null);
$route = self::$mysql->getRow(self::$mysql->query(sprintf("SELECT autokid,route_flag,real_table,hittime,hosts_list FROM %s WHERE table_name='mirror' AND route_flag=%d LIMIT 1", $where['table'], Router::FLAG_PRE_IMPORT)));
$this->assertEquals(0, $route['hittime']);
$this->assertEquals('$', $route['hosts_list']);
$real_table = $route['real_table'];
// XXX: 数据装完
$this->assertEquals(1, Router::effect('mirror', null, $route['real_table'], '1,2'));
$this->assertEquals(array('hosts_list' => '1,2,$', 'route_flag' => Router::FLAG_IMPORT_END), self::$mysql->getRow(self::$mysql->query(sprintf("SELECT hosts_list,route_flag FROM %s WHERE table_name='mirror' AND real_table='%s' AND autokid = %u", $where['table'], $route['real_table'], $route['autokid']))));
$this->assertEquals(array(), Router::get('mirror'));
// xxx: 模拟路由生效
Router::flush();
$route = Router::get('mirror', null, true);
$this->assertEquals(1, count($route));
$route = reset($route);
$this->assertTrue(0 < $route['mtime']);
$this->assertEquals('1,2', $route['hosts']);
$this->assertEquals($real_table, $route['table']);
Router::removeAllCache();
$query = sprintf('SELECT hittime FROM %s WHERE %s AND route_flag=%d ORDER BY autokid DESC LIMIT 1', $where['table'], $where['where'], Router::FLAG_NORMAL_USE);
$this->assertEquals(intval(time() / 2), intval(self::$mysql->getOne(self::$mysql->query($query)) / 2));
}
public function test_should_import_consist_works_fine()
{
$hosts = self::$mysql->getAll(self::$mysql->query(sprintf('SELECT host_id, host_name FROM %shost_list WHERE host_type = %d', self::$mysql->option('prefix'), Server::TYPE_REALITY)));
$create = "CREATE TABLE IF NOT EXISTS test.test_a (\n id int(10) unsigned not null auto_increment primary key,\n num1 int(10) not null default 0,\n char1 varchar(15) not null default ''\n ) ENGINE=MYISAM DEFAULT CHARSET=UTF8";
$ids = array();
foreach ($hosts as $server) {
$mysql = Server::instance($server['host_name'])->getlink();
$mysql->query('DROP TABLE IF EXISTS test.test_a');
$mysql->query($create);
$this->assertEquals(3, $mysql->query("INSERT INTO test.test_a (id,num1,char1) VALUES (1, 2, 'bbb'),(2,3,'cccc'),(3,4,'dddd')"));
$ids[] = (int) $server['host_id'];
}
$this->assertEquals(1, self::$mysql->query(sprintf("INSERT INTO %sroute_info (real_table,hosts_list,table_name,modtime,route_flag)" . " VALUES ('test.test_a', '%s', 'test',%d,%d)", self::$mysql->option('prefix'), implode(',', $ids), time(), \Myfox\App\Model\Router::FLAG_IMPORT_END)));
\Myfox\App\Setting::set('monitor_consist_check', date('Y-m-d H:i:s', time() - 3600));
$this->assertEquals(array(), Monitor::import_consist_monitor());
$server = reset($hosts);
Server::instance($server['host_name'])->getlink()->query(sprintf('DELETE FROM test.test_a ORDER BY id ASC LIMIT 1'));
\Myfox\App\Setting::set('monitor_consist_check', date('Y-m-d H:i:s', time() - 3600));
$result = Monitor::import_consist_monitor();
$this->assertTrue(!empty($result));
$result = reset($result);
$this->assertTrue(!empty($result['checks']));
}
/**
* 获取某条分片的信息
* @param {String} split 路由分片表名
* @param {String} table 某张具体表的路由信息
* @return {Array} 此路由分片表中所有的分片信息
*/
private function get_route_infos($table = '')
{
$sql = '';
if (empty($table)) {
$sql = sprintf('SELECT autokid,table_name,real_table,hosts_list FROM %s%s
WHERE route_flag >= %s AND route_flag < %s AND modtime >= %s', $this->mysql->escape($this->mysql->option('prefix')), $this->routetab, '300', '400', strtotime(Setting::get('unique_last_check')));
} else {
$sql = sprintf("SELECT autokid,addtime,real_table,hosts_list FROM %s%s \n WHERE table_name = '%s' AND route_flag >= %s AND route_flag < %s AND addtime > %s", $this->mysql->escape($this->mysql->option('prefix')), $this->routetab, $this->mysql->escape($table), '300', '400', time() - $this->option['interval']);
}
return $this->mysql->getAll($this->mysql->query($sql));
}
/**
* 计算路由
*
* @access private
* @return Mixture
*/
private function insert($detail = array())
{
if (self::MIRROR == $this->table->get('route_type')) {
$hosts = self::$hostall;
$backup = count($hosts);
$chunks = array(array(array('data' => '', 'size' => empty($detail[0]['count']) ? 0 : $detail[0]['count'])));
} else {
$hosts = self::$onlines;
$backup = min(max(1, $this->table->get('backups')), count(self::$onlines));
$bucket = new \Myfox\App\Bucket($this->table->get('split_threshold', 2000000), $this->table->get('split_drift', 0.2));
foreach ((array) $detail as $route) {
$bucket->push($this->filter($route['field']), $route['count']);
}
$chunks = $bucket->allot();
}
$counts = count($hosts);
$skips = max(1, (int) ($counts / $backup));
$last = (int) Setting::get('last_assign_host');
$bucket = array();
$cursor = (int) Setting::get('table_route_count', $this->tbname);
$dbnums = (int) Setting::get('table_real_count', $this->tbname);
foreach ($chunks as $items) {
$ns = array();
for ($i = 0; $i < $backup; $i++) {
$ns[] = $hosts[($i * $skips + $last) % $counts];
}
$ns = implode(',', $ns);
$tb = sprintf('%s_%d.t_%d_%d', $this->tbname, $dbnums % self::TABLES_PER_DB, $this->table->get('autokid'), self::MIRROR === $this->table->get('route_type') ? $cursor : $cursor % 3);
foreach ($items as $it) {
$bucket[$it['data']][] = array('rows' => $it['size'], 'hosts' => $ns, 'table' => $tb);
}
$cursor++;
$last++;
}
Setting::set('last_assign_host', $last % $counts);
Setting::set('table_route_count', sprintf('IF(cfgvalue + 0 > %d, cfgvalue, %d)', $cursor, $cursor), $this->tbname, false);
$time = time();
self::$mysql->begin();
foreach ($bucket as $key => $slice) {
$sign = $this->sign($key);
$key = self::$mysql->escape($key);
$values = array();
foreach ($slice as $rt) {
$values[] = sprintf("(%d,0,%d,%d,'%s','%s','%s\$','%s')", $sign, self::FLAG_PRE_IMPORT, $time, $this->tbname, $key, '', $rt['table']);
}
$rt = false !== self::$mysql->query(sprintf("UPDATE %s SET route_flag=IF(route_flag=%d,%d,%d) WHERE route_sign=%u AND table_name='%s' AND route_text='%s'", self::table($sign), self::FLAG_NORMAL_USE, self::FLAG_PRE_RESHIP, self::FLAG_IS_DELETED, $sign, $this->tbname, $key)) && false !== self::$mysql->query(sprintf('INSERT INTO %s (route_sign,is_archive,route_flag,addtime,table_name,route_text,hosts_list,real_table) VALUES %s', self::table($sign), implode(',', $values)));
if (empty($rt)) {
self::$mysql->rollback();
return false;
}
}
self::$mysql->commit();
return $bucket;
}
public function test_add_new_split_works_fine()
{
$this->test_set_split_works_fine();
sleep(3);
$this->mysql->query(sprintf("insert into %sroute_info (table_name,addtime,modtime,real_table,hosts_list,route_flag) values ('testTable',%s,%s,'meta_myfox_config.test_table_real3','99\$',300)", $this->mysql->option("prefix"), time(), time()));
$this->mysql->query("insert into test_table_real3 (key1,key2) values ('a',2)");
$this->mysql->query("insert into test_table_real3 (key1,key2) values ('b',5)");
$originalModtime = Setting::get('unique_last_check');
$uniqueBuilder = new \Myfox\App\Worker\Uniquebuilder();
$uniqueBuilder->execute();
$res = $this->mysql->getAll($this->mysql->query(sprintf("select unique_key from %sroute_info where table_name = 'testTable' and real_table = 'meta_myfox_config.test_table_real3'", $this->mysql->option("prefix"))));
$this->assertEquals(array('unique_key' => 'id;key1;key2$'), $res[0]);
$res = $this->mysql->getAll($this->mysql->query(sprintf("select cfgvalue,modtime from %ssettings where cfgname = 'unique_last_check'", $this->mysql->option("prefix"))));
$this->assertFalse($originalModtime === $res[0]['modtime']);
}
/**
* 数据装载一致性监控
*
* @access public static
* @return Mixture
*/
public static function import_consist_monitor()
{
$mysql = \Myfox\Lib\Mysql::instance('default');
$tables = $mysql->getAll($mysql->query(sprintf("SHOW TABLES LIKE '%sroute_info%%'", $mysql->option('prefix'))));
$status = array();
$offset = (int) strtotime(Setting::get('monitor_consist_check'));
foreach ($tables as $table) {
$table = end($table);
$split = $mysql->getAll($mysql->query(sprintf('SELECT real_table,hosts_list,route_text,table_name FROM %s WHERE modtime >= %d AND route_flag IN (%d, %d)', $table, $offset, Router::FLAG_IMPORT_END, Router::FLAG_NORMAL_USE)));
if (empty($split)) {
continue;
}
foreach ($split as $row) {
$server = array();
foreach (explode(',', trim($row['hosts_list'], '{},$')) as $id) {
$server[] = self::hostname($id);
}
$result = Consist::check($row['real_table'], $server);
if (is_array($result)) {
$status[] = array('table' => $row['table_name'], 'route' => $row['route_text'], 'bucket' => $row['real_table'], 'checks' => $result);
}
}
}
Setting::set('monitor_consist_check', date('Y-m-d H:i:s'));
return $status;
}
/**
* 数据装载状态查询
*
* @access protected
* @return void
*/
protected function actionIndex($param, $post = null)
{
self::output(0, 'OK', json_encode(array('last_date' => \Myfox\App\Setting::get('last_date'))));
}
protected function actionLastdate($param)
{
echo json_encode(array('group_name' => '__global__', 'last_date' => \Myfox\App\Setting::get('myfox_last_date')));
}
public function test_should_ready_task_works_fine()
{
$task = new \Myfox\App\Task\Ready(-1, array());
$this->assertEquals(Task::IGNO, $task->execute());
\Myfox\App\Setting::set('last_date', '2011-10-01');
$task = new \Myfox\App\Task\Ready(-1, array('thedate' => '2011-10-01', 'priority' => 1));
$this->assertEquals(Task::IGNO, $task->execute());
self::$mysql->query(sprintf('TRUNCATE TABLE %stask_queque', self::$mysql->option('prefix')));
self::$mysql->query(sprintf('INSERT INTO %stask_queque (autokid,priority,task_flag,task_type) VALUES ' . "(1,21,%d,'import'),(2,20,%d,'import'),(3,20,%d,'import'),(4,19,%d,'lalallala'),(101,1,%d,'import')", self::$mysql->option('prefix'), \Myfox\App\Queque::FLAG_WAIT, \Myfox\App\Queque::FLAG_NEW, \Myfox\App\Queque::FLAG_IGNO, \Myfox\App\Queque::FLAG_LOCK, \Myfox\App\Queque::FLAG_WAIT));
\Myfox\App\Setting::set('last_date', '2011-10-01');
$task = new \Myfox\App\Task\Ready(100, array('thedate' => '2011-10-02', 'priority' => 20));
$this->assertEquals(Task::FAIL, $task->execute());
$this->assertContains('Waiting for 1 import task(s)', $task->lastError());
self::$mysql->query(sprintf('TRUNCATE TABLE %stask_queque', self::$mysql->option('prefix')));
\Myfox\App\Setting::set('last_date', '2011-10-01');
$task = new \Myfox\App\Task\Ready(100, array('thedate' => '2011-10-02', 'priority' => 20));
$this->assertEquals(Task::SUCC, $task->execute());
}