/**
  * 清理失效的路由
  *
  * @access public static
  * @return void
  */
 public static function clean_invalidated_routes()
 {
     $mysql = \Myfox\Lib\Mysql::instance('default');
     $query = sprintf("SHOW TABLES LIKE '%sroute_info%%'", $mysql->option('prefix'));
     foreach ((array) $mysql->getAll($mysql->query($query)) as $table) {
         $table = end($table);
         $query = sprintf('SELECT autokid,hosts_list,real_table,route_text FROM %s WHERE route_flag IN (%d) AND modtime < %d', $table, Router::FLAG_IS_DELETED, time() - 100000);
         $ids = array();
         foreach ((array) $mysql->getAll($mysql->query($query)) as $row) {
             $hosts = trim($row['hosts_list'], '{}$');
             if (!empty($hosts)) {
                 $where = array();
                 foreach (Router::parse($row['route_text']) as $k => $v) {
                     $where[] = sprintf('%s = %s', $k, $v);
                 }
                 $info = array('host' => $hosts, 'path' => $row['real_table'], 'where' => implode(' AND ', $where));
                 if (!Queque::instance()->insert('delete', $info, 0, array('adduser' => 'cleaner'))) {
                     continue;
                 }
             }
             $ids[] = (int) $row['autokid'];
         }
         $mysql->query(sprintf('DELETE FROM %s WHERE (route_flag = %d AND addtime < %d)%s', $table, Router::FLAG_PRE_IMPORT, time() - 10 * 86400, empty($ids) ? '' : sprintf(' OR (autokid IN (%s))', implode(',', $ids))));
     }
 }
 public function test_should_close_race_queque_works_fine()
 {
     $queque = Queque::instance();
     $this->assertEquals(null, $queque->fetch());
     $this->assertTrue($queque->insert('test ', array('src' => 'http://www.taobao.com'), 1, array('trytimes' => 2, 'adduser' => 'unittest', 'priority' => 201, 'openrace' => 0)));
     $this->assertEquals(null, $queque->fetch(1, 2));
     $this->assertEquals(array('id' => 1, 'type' => 'test', 'status' => '', 'info' => json_encode(array('src' => 'http://www.taobao.com'))), $queque->fetch(1, 1));
 }
 public function test_should_queque_fetch_by_type_works_fine()
 {
     $queque = Queque::instance();
     $this->assertTrue($queque->insert('test ', array('src' => 'http://www.taobao.com'), 1, array('trytimes' => 2, 'adduser' => 'unittest', 'priority' => 201)));
     $this->assertTrue($queque->insert('test2', array('src' => 'http://www.taobao.com'), 1, array('trytimes' => 2, 'adduser' => 'unittest', 'priority' => 200)));
     $task = $queque->fetch(1, 2, Queque::FLAG_WAIT, 'test');
     $this->assertEquals(array('id' => 1, 'type' => 'test', 'status' => '', 'info' => json_encode(array('src' => 'http://www.taobao.com'))), $task);
 }
 public function test_should_undefined_task_type_ignore_works_fine()
 {
     $this->assertTrue(Queque::instance()->insert('i am not defined', array('src' => 'http://www.taobao.com'), 1, array('trytimes' => 2, 'adduser' => 'unittest', 'priority' => 201)));
     $worker = new Processor(array('n' => 1));
     $this->assertTrue($worker->execute());
     $this->assertEquals(1, $worker->interval());
     $queque = self::$mysql->getRow(self::$mysql->query(sprintf('SELECT * FROM %stask_queque LIMIT 1', self::$mysql->option('prefix', ''))));
     $this->assertEquals(1, $queque['trytimes']);
     $this->assertEquals(Queque::FLAG_IGNO, $queque['task_flag']);
     $this->assertContains("Undefined task_type named as 'i am not defined'", $queque['last_error']);
 }
Example #5
0
 /**
  * 完成后任务解锁
  *
  * @access public
  * @return Boolean true or false
  */
 private static function unlock($id, $flag = Queque::FLAG_DONE, $option = null, $comma = null)
 {
     return Queque::instance()->update($id, array('trytimes' => sprintf('IF(task_flag=%d,trytimes-1,trytimes)', Queque::FLAG_LOCK), 'endtime' => sprintf("IF(task_flag=%d,'%s',endtime)", Queque::FLAG_LOCK, date('Y-m-d H:i:s')), 'task_flag' => $flag) + (array) $option, array('trytimes' => true, 'endtime' => true, 'task_flag' => true) + (array) $comma);
 }
Example #6
0
 /**
  * 切分好的分片数据装载, 外部切分程序调用
  *
  * @access protected
  * @return void
  */
 protected function actionQueque($gets, $data)
 {
     if (false === ($priority = self::priority())) {
         self::output(1100, 'Access Denied.');
         return false;
     }
     $table = self::vars('table', $data, $gets);
     if (empty($table)) {
         self::output(1200, 'Param "table" is required.');
         return false;
     }
     foreach (array('file', 'route', 'bucket', 'hosts') as $key) {
         if (empty($data[$key])) {
             self::output(1201, sprintf('Param "%s" is required in post data.', $key));
             return false;
         }
     }
     $info = array('table' => $table, 'route' => $data['route'], 'file' => $data['file'], 'bucket' => $data['bucket'], 'hosts' => $data['hosts']);
     $ipaddr = Context::userip();
     $agent = isset($param['agent']) ? (int) $param['agent'] : ip2long($ipaddr);
     $option = array('priority' => $priority, 'trytimes' => 3, 'task_flag' => \Myfox\App\Queque::FLAG_WAIT, 'adduser' => sprintf('web:%s', $ipaddr));
     $queque = \Myfox\App\Queque::instance();
     if (!$queque->insert('import', $info, $agent, $option)) {
         self::output(1300, sprintf('Inner Error (%s).', $queque->lastError()));
         return false;
     }
     self::output(0, 'OK', array('id' => $queque->lastId()));
 }
Example #7
0
 /**
  * OPTIONS:
  * ----------------------------------------------------------------------
  * @file    : complete file url, such as "ftp://*****:*****@hostname/path"
  * @table   : name of the logic table
  * @route   : route value of the file, ex: thedate=20111001,cid=210
  * @lines   :
  * ---------------------------------------------------------------------
  */
 public function execute()
 {
     if (!$this->isReady('table', 'route', 'lines', 'file', 'priority')) {
         return self::IGNO;
     }
     $table = Table::instance($this->option('table'));
     if (!$table->get('autokid')) {
         $this->setError(sprintf('Undefined table named as "%s".', $this->option('table')));
         return self::IGNO;
     }
     try {
         $routes = Router::set($this->option('table'), array(array('field' => Router::parse($this->option('route'), $this->option('table')), 'count' => (int) $this->option('lines'))));
         if (!is_array($routes)) {
             $this->setError('route failed.');
             return self::FAIL;
         }
         $config = Config::instance('default');
         $fname = Fileset::getfile($this->option('file'), $config->get('path/download'));
         if (empty($fname)) {
             $this->setError(sprintf('getfile:%s', Fileset::lastError()));
             return self::FAIL;
         }
         $splits = array();
         foreach ($routes as $key => $bucket) {
             foreach ($bucket as $item) {
                 $splits[] = $item['rows'];
             }
         }
         $fsplit = new Fsplit($fname, "\n", 16777216);
         $chunks = $fsplit->split($splits, $config->get('path/filesplit'));
         if (empty($chunks)) {
             $this->setError(sprintf('fsplit failed,%s', $fsplit->lastError()));
             return self::FAIL;
         }
         if (preg_match('/^\\w+:/i', $this->option('file'))) {
             @unlink($fname);
         }
         $result = array();
         $queque = Queque::instance();
         foreach ($routes as $key => $bucket) {
             foreach ($bucket as $item) {
                 $fname = array_shift($chunks);
                 if (empty($fname)) {
                     break 2;
                 }
                 $info = array('table' => $this->option('table'), 'route' => $key, 'file' => $fname, 'bucket' => $item['table'], 'hosts' => $item['hosts']);
                 $option = array('openrace' => 0, 'priority' => (int) $this->option('priority'), 'trytimes' => 3, 'task_flag' => Queque::FLAG_WAIT, 'adduser' => 'rsplit');
                 if (!$queque->insert('import', $info, -1, $option)) {
                     $this->setError(sprintf('queque: %s', $queque->lastError()));
                     return self::FAIL;
                 }
                 $result[] = $queque->lastId();
             }
         }
         $this->result = implode(',', $result);
     } catch (\Exception $e) {
         $this->setError($e->getMessage());
         return self::FAIL;
     }
     return self::SUCC;
 }