public function generate(swoole_process $worker) { $job = $worker->sitemap_job; //先判断这个job是否为空; if (empty($job)) { exit(0); } //需要新的mysql连接 $config = (array) Cola::config()->get('_db') + array('adapter' => 'Mysql', 'params' => array()); $db = Cola_Com_Db::factory($config); $this->cola_db = $db; //取出keys $files = array_keys($job); foreach ($files as $file) { //open this file , w+;不存在就创建,并且truncate文件 $fileHandler = fopen($file, 'w+'); $this->writeHeader($fileHandler); $low_id = $job[$file]['low']; $high_id = $job[$file]['high']; // count,从low到high有多少条记录 $count = $this->countPic($low_id, $high_id); $times = $count / self::BUFFER_ROW; //如果times为0,也会运行一次$offset=0,可以达到目的 foreach (range(0, $times) as $index) { $offset = $index * self::BUFFER_ROW; $pics = $this->selectPics($offset, $low_id, $high_id); $buffer_string = $this->fetchView($pics); fwrite($fileHandler, $buffer_string); } $this->writeFooter($fileHandler); fflush($fileHandler); fclose($fileHandler); } }
public function runAction() { //get log id $pic_id = intval($this->getLogId()); //die(var_dump($pic_id)); //取得为标签图片(id, url) $workerNum = 5; $limit = 20; $totalCount = $this->countUntaggedPic($pic_id); $partTotalCount = $totalCount / $workerNum; $jobs = $this->calculateJob($totalCount, $workerNum, $pic_id); var_dump($jobs); if ($jobs) { foreach ($jobs as $index => $range) { $pid = pcntl_fork(); if (!$pid) { //子进程 $endId = $range['endId']; if (file_exists(ROOT_DIR . "/sbin/child_pic_id_{$index}.log")) { $pid_log_file = fopen("child_pic_id_{$index}.log", 'r+'); $startId = (int) file_get_contents(ROOT_DIR . "/sbin/child_pic_id_{$index}.log"); } else { //log不存在 $pid_log_file = fopen("child_pic_id_{$index}.log", 'w+'); $startId = $range['startId']; } //新建一个mysql连接,抛弃原来的 $config = (array) Cola::config()->get('_db') + array('adapter' => 'Mysql', 'params' => array()); $db = Cola_Com_Db::factory($config); $this->cola_db = $db; //$this->dbs["DB_".$index] = $db; $times = $partTotalCount / $limit + 1; for ($i = 0; $i < $times; $i++) { $offset = $i * $limit; $pics = $this->getUntaggedPicWithEnd($startId, $endId, $offset, $limit); foreach ($pics as $pic) { $id = $pic['id']; $pic_url = $this->getPicUrl($pic['orig_path']); $this->handle($pic_url, $id); } //log pic_id rewind($pid_log_file); fwrite($pid_log_file, $id); fflush($pid_log_file); } //子进程退出时需要关闭资源 $this->cola_db->close(); fclose($pid_log_file); echo "Child runs to end"; } } //父进程等待子进程结束 while (pcntl_waitpid(0, $status) != -1) { $status = pcntl_wexitstatus($status); echo "Child {$status} completed\n"; } } else { //没有jobs,不需要多进程 $times = $totalCount / $limit + 1; for ($i = 0; $i < $times; $i++) { $offset = $i * $limit; $pics = $this->getUntaggedPic($pic_id, $offset, $limit); foreach ($pics as $pic) { $id = $pic['id']; $pic_url = $this->getPicUrl($pic['orig_path']); $this->handle($pic_url, $id); } $this->logId($id); } } }
public static function db($config) { return Cola_Com_Db::factory($config); }
/** * Slave MySQL Adapter * * @param string $name * @return Cola_Com_Db_Abstract */ public function slave($name = null) { if ($this->_slave) { return $this->_slave; } if (is_null($name) || empty($this->_config['slave'][$name])) { $name = array_rand($this->_config['slave']); } $params = $this->_config['slave'][$name]; $config = array('adapter' => $this->_config['adapter'], 'params' => $params); $this->_slave = Cola_Com_Db::factory($config); $this->_mysql = $this->_slave; return $this->_slave; }
public function push(swoole_process $worker){ $job = null; if(isset($worker->sitemap_job)){ $job = $worker->sitemap_job; } //先判断这个job是否为空; if(empty($job)){ exit(0); } //需要新的mysql连接 $config = (array)Cola::config()->get('_db') + array('adapter' => 'Mysql', 'params' => array()); $db = Cola_Com_Db::factory($config); $this->cola_db = $db; //取出keys $files = array_keys($job); foreach($files as $file){ $low_id = $job[$file]['low']; $high_id = $job[$file]['high']; // count,从low到high有多少条记录 $count = $this->countPic($low_id, $high_id); $times = $count/self::BUFFER_ROW; //如果times为0,也会运行一次$offset=0,可以达到目的 foreach(range(0, $times) as $index){ $offset = $index*self::BUFFER_ROW; $pics = $this->selectPics($offset, $low_id, $high_id); $this->pushToBaidu($pics); } } }
/** * Connect db from config * * @param array $config * @param string $regName * @return Cola_Com_Db */ public function db($name = null) { if (empty($name)) { $name = $this->_db; } if (is_array($name)) { $config = (array) $name + array('adapter' => 'Mysql', 'params' => array()); return Cola_Com_Db::factory($config); } if ($db = Cola::reg($name)) { return $db; } $config = (array) Cola::config($name) + array('adapter' => 'Mysql', 'params' => array()); $db = Cola_Com_Db::factory($config); Cola::reg($name, $db); return $db; }