public function test_should_get_file_from_scp_works_fine()
{
#exec('ssh-keygen -t rsa');
/* 信任关系打通 设置免登 */
exec('ssh-copy-id -i ~/.ssh/id_rsa.pub 10.232.128.63');
#$user = get_current_user();
$user = array_shift(explode(' ', trim(exec('who am i'))));
$fname = sprintf('Myfox_scp_test%s.txt', getmypid());
$lpath = sprintf('/tmp/myfox/download/Myfox_scp_test%s.txt', getmypid());
$rpath = sprintf('/home/%s/Myfox_scp_test%s.txt', $user, getmypid());
$url = sprintf('scp://%s@10.232.128.63%s', $user, $rpath);
exec(sprintf('ssh %s@10.232.128.63 "rm -rf %s"', $user, $rpath));
@unlink($lpath);
$this->assertEquals(false, Fileset::getfile($url));
$this->assertContains('No such file or directory', Fileset::lastError());
exec(sprintf('ssh %s@10.232.128.63 "touch %s"', $user, $rpath));
sleep(2);
$this->assertEquals($lpath, Fileset::getfile($url));
$scp = new Scp(array('scheme' => 'scp', 'host' => '10.232.128.63', 'user' => $user, 'path' => $rpath));
$this->assertEquals(false, $scp->isChange($lpath));
exec(sprintf('ssh %s@10.232.128.63 "echo \\"test data\\" >> %s"', $user, $rpath));
$this->assertEquals($lpath, Fileset::getfile($url));
$content = file_get_contents($lpath);
$this->assertContains('test data', $content);
}
public function test_should_get_file_from_ftp_works_fine()
{
$this->assertEquals(false, Fileset::getfile('ftp://www.baidu.com/aa.txt'));
$this->assertContains('Connect failed with the host as "www.baidu.com" on port 21', Fileset::lastError());
$fname = Fileset::getfile('ftp://ftp.adobe.com/license.txt?timeout=5');
$this->assertEquals(sprintf('%s/license.txt', Fileset::TEMP_PATH), $fname);
}
public function execute()
{
/**
* OPTIONS:
* ----------------------------------------------------------------------
* @file : complete file url, such as "ftp://*****:*****@hostname/path"
* @table : name of the logic table
* @route : route value of the file, ex: thedate=20111001,cid=210
* @bucket :
* @hosts :
* ---------------------------------------------------------------------
*/
$this->pools = array();
if (!$this->isReady('table', 'route', 'file', 'bucket', 'hosts')) {
return self::IGNO;
}
$table = Table::instance($this->option('table'));
if (!$table->get('autokid')) {
$this->setError(sprintf('Undefined table named as "%s".', $this->option('table')));
return self::IGNO;
}
$fname = null;
$this->route = Router::parse($this->option('route'));
$this->result = array_flip(explode(',', $this->status));
foreach (array_unique(explode(',', trim($this->option('hosts'), '{}$'))) as $host) {
if (isset($this->result[$host])) {
continue;
}
empty($fname) && ($fname = Fileset::getfile($this->option['file']));
if (empty($fname)) {
$this->setError(Fileset::lastError());
return self::FAIL;
}
$config = \Myfox\Lib\Config::instance('default');
$ibservers = array_unique(explode(',', trim($config->get('ib/servers'))));
$ibtables = array_unique(explode(',', trim($config->get('ib/tables'))));
if (in_array($host, $ibservers) && in_array($this->option('table'), $ibtables)) {
$this->onehost($host, $fname, 'BRIGHTHOUSE');
} else {
$this->onehost($host, $fname, 'MYISAM');
}
}
return empty($this->pools) ? self::FAIL : self::WAIT;
}
/**
* OPTIONS:
* ----------------------------------------------------------------------
* @file : complete file url, such as "ftp://*****:*****@hostname/path"
* @table : name of the logic table
* @route : route value of the file, ex: thedate=20111001,cid=210
* @lines :
* ---------------------------------------------------------------------
*/
public function execute()
{
if (!$this->isReady('table', 'route', 'lines', 'file', 'priority')) {
return self::IGNO;
}
$table = Table::instance($this->option('table'));
if (!$table->get('autokid')) {
$this->setError(sprintf('Undefined table named as "%s".', $this->option('table')));
return self::IGNO;
}
try {
$routes = Router::set($this->option('table'), array(array('field' => Router::parse($this->option('route'), $this->option('table')), 'count' => (int) $this->option('lines'))));
if (!is_array($routes)) {
$this->setError('route failed.');
return self::FAIL;
}
$config = Config::instance('default');
$fname = Fileset::getfile($this->option('file'), $config->get('path/download'));
if (empty($fname)) {
$this->setError(sprintf('getfile:%s', Fileset::lastError()));
return self::FAIL;
}
$splits = array();
foreach ($routes as $key => $bucket) {
foreach ($bucket as $item) {
$splits[] = $item['rows'];
}
}
$fsplit = new Fsplit($fname, "\n", 16777216);
$chunks = $fsplit->split($splits, $config->get('path/filesplit'));
if (empty($chunks)) {
$this->setError(sprintf('fsplit failed,%s', $fsplit->lastError()));
return self::FAIL;
}
if (preg_match('/^\\w+:/i', $this->option('file'))) {
@unlink($fname);
}
$result = array();
$queque = Queque::instance();
foreach ($routes as $key => $bucket) {
foreach ($bucket as $item) {
$fname = array_shift($chunks);
if (empty($fname)) {
break 2;
}
$info = array('table' => $this->option('table'), 'route' => $key, 'file' => $fname, 'bucket' => $item['table'], 'hosts' => $item['hosts']);
$option = array('openrace' => 0, 'priority' => (int) $this->option('priority'), 'trytimes' => 3, 'task_flag' => Queque::FLAG_WAIT, 'adduser' => 'rsplit');
if (!$queque->insert('import', $info, -1, $option)) {
$this->setError(sprintf('queque: %s', $queque->lastError()));
return self::FAIL;
}
$result[] = $queque->lastId();
}
}
$this->result = implode(',', $result);
} catch (\Exception $e) {
$this->setError($e->getMessage());
return self::FAIL;
}
return self::SUCC;
}
