public function generate(swoole_process $worker)
{
$job = $worker->sitemap_job;
//先判断这个job是否为空；
if (empty($job)) {
exit(0);
}
//需要新的mysql连接
$config = (array) Cola::config()->get('_db') + array('adapter' => 'Mysql', 'params' => array());
$db = Cola_Com_Db::factory($config);
$this->cola_db = $db;
//取出keys
$files = array_keys($job);
foreach ($files as $file) {
//open this file ， w+；不存在就创建，并且truncate文件
$fileHandler = fopen($file, 'w+');
$this->writeHeader($fileHandler);
$low_id = $job[$file]['low'];
$high_id = $job[$file]['high'];
// count，从low到high有多少条记录
$count = $this->countPic($low_id, $high_id);
$times = $count / self::BUFFER_ROW;
//如果times为0，也会运行一次$offset=0，可以达到目的
foreach (range(0, $times) as $index) {
$offset = $index * self::BUFFER_ROW;
$pics = $this->selectPics($offset, $low_id, $high_id);
$buffer_string = $this->fetchView($pics);
fwrite($fileHandler, $buffer_string);
}
$this->writeFooter($fileHandler);
fflush($fileHandler);
fclose($fileHandler);
}
}
public function runAction()
{
//get log id
$pic_id = intval($this->getLogId());
//die(var_dump($pic_id));
//取得为标签图片(id, url)
$workerNum = 5;
$limit = 20;
$totalCount = $this->countUntaggedPic($pic_id);
$partTotalCount = $totalCount / $workerNum;
$jobs = $this->calculateJob($totalCount, $workerNum, $pic_id);
var_dump($jobs);
if ($jobs) {
foreach ($jobs as $index => $range) {
$pid = pcntl_fork();
if (!$pid) {
//子进程
$endId = $range['endId'];
if (file_exists(ROOT_DIR . "/sbin/child_pic_id_{$index}.log")) {
$pid_log_file = fopen("child_pic_id_{$index}.log", 'r+');
$startId = (int) file_get_contents(ROOT_DIR . "/sbin/child_pic_id_{$index}.log");
} else {
//log不存在
$pid_log_file = fopen("child_pic_id_{$index}.log", 'w+');
$startId = $range['startId'];
}
//新建一个mysql连接，抛弃原来的
$config = (array) Cola::config()->get('_db') + array('adapter' => 'Mysql', 'params' => array());
$db = Cola_Com_Db::factory($config);
$this->cola_db = $db;
//$this->dbs["DB_".$index] = $db;
$times = $partTotalCount / $limit + 1;
for ($i = 0; $i < $times; $i++) {
$offset = $i * $limit;
$pics = $this->getUntaggedPicWithEnd($startId, $endId, $offset, $limit);
foreach ($pics as $pic) {
$id = $pic['id'];
$pic_url = $this->getPicUrl($pic['orig_path']);
$this->handle($pic_url, $id);
}
//log pic_id
rewind($pid_log_file);
fwrite($pid_log_file, $id);
fflush($pid_log_file);
}
//子进程退出时需要关闭资源
$this->cola_db->close();
fclose($pid_log_file);
echo "Child runs to end";
}
}
//父进程等待子进程结束
while (pcntl_waitpid(0, $status) != -1) {
$status = pcntl_wexitstatus($status);
echo "Child {$status} completed\n";
}
} else {
//没有jobs，不需要多进程
$times = $totalCount / $limit + 1;
for ($i = 0; $i < $times; $i++) {
$offset = $i * $limit;
$pics = $this->getUntaggedPic($pic_id, $offset, $limit);
foreach ($pics as $pic) {
$id = $pic['id'];
$pic_url = $this->getPicUrl($pic['orig_path']);
$this->handle($pic_url, $id);
}
$this->logId($id);
}
}
}
public static function db($config)
{
return Cola_Com_Db::factory($config);
}
/**
* Slave MySQL Adapter
*
* @param string $name
* @return Cola_Com_Db_Abstract
*/
public function slave($name = null)
{
if ($this->_slave) {
return $this->_slave;
}
if (is_null($name) || empty($this->_config['slave'][$name])) {
$name = array_rand($this->_config['slave']);
}
$params = $this->_config['slave'][$name];
$config = array('adapter' => $this->_config['adapter'], 'params' => $params);
$this->_slave = Cola_Com_Db::factory($config);
$this->_mysql = $this->_slave;
return $this->_slave;
}
public function push(swoole_process $worker){
$job = null;
if(isset($worker->sitemap_job)){
$job = $worker->sitemap_job;
}
//先判断这个job是否为空；
if(empty($job)){
exit(0);
}
//需要新的mysql连接
$config = (array)Cola::config()->get('_db') + array('adapter' => 'Mysql', 'params' => array());
$db = Cola_Com_Db::factory($config);
$this->cola_db = $db;
//取出keys
$files = array_keys($job);
foreach($files as $file){
$low_id = $job[$file]['low'];
$high_id = $job[$file]['high'];
// count，从low到high有多少条记录
$count = $this->countPic($low_id, $high_id);
$times = $count/self::BUFFER_ROW;
//如果times为0，也会运行一次$offset=0，可以达到目的
foreach(range(0, $times) as $index){
$offset = $index*self::BUFFER_ROW;
$pics = $this->selectPics($offset, $low_id, $high_id);
$this->pushToBaidu($pics);
}
}
}
/**
* Connect db from config
*
* @param array $config
* @param string $regName
* @return Cola_Com_Db
*/
public function db($name = null)
{
if (empty($name)) {
$name = $this->_db;
}
if (is_array($name)) {
$config = (array) $name + array('adapter' => 'Mysql', 'params' => array());
return Cola_Com_Db::factory($config);
}
if ($db = Cola::reg($name)) {
return $db;
}
$config = (array) Cola::config($name) + array('adapter' => 'Mysql', 'params' => array());
$db = Cola_Com_Db::factory($config);
Cola::reg($name, $db);
return $db;
}
