public function it_can_sadd()
{
$this->redis->sAdd('test', 'test')->shouldBeCalled();
$this->sAdd('test', ['test']);
$this->redis->sAdd('test', 'test')->willThrow(new \RedisException());
$this->shouldThrow(DriverException::class)->during('sAdd', ['test', ['test']]);
}
/**
* @inheritdoc
*/
public function gauge($bucket, $value)
{
$time = $this->syncedTime();
$granularities = $this->getGranularities();
foreach ($granularities as $granularity => $settings) {
$key = $this->getKey($bucket, 'gauges', $granularity, $settings, $time);
$field = $this->getField($settings, $time);
$this->redis->hSet($key, $field, $value);
$this->redis->expireAt($key, $time + $settings['ttl']);
}
$this->redis->sAdd('buckets', $bucket);
$this->redis->sAdd(sprintf('types:%s', $bucket), 'gauges');
}
/**
* Saves data in the cache.
*
* @param string $entryIdentifier An identifier for this specific cache entry
* @param string $data The data to be stored
* @param array $tags Tags to associate with this cache entry. If the backend does not support tags, this option can be ignored.
* @param integer $lifetime Lifetime of this cache entry in seconds. If NULL is specified, the default lifetime is used. "0" means unlimited lifetime.
* @throws \RuntimeException
* @return void
* @api
*/
public function set($entryIdentifier, $data, array $tags = [], $lifetime = null)
{
if ($this->isFrozen()) {
throw new \RuntimeException(sprintf('Cannot add or modify cache entry because the backend of cache "%s" is frozen.', $this->cacheIdentifier), 1323344192);
}
if ($lifetime === null) {
$lifetime = $this->defaultLifetime;
}
$setOptions = [];
if ($lifetime > 0) {
$setOptions['ex'] = $lifetime;
}
$this->redis->multi();
$result = $this->redis->set($this->buildKey('entry:' . $entryIdentifier), $this->compress($data), $setOptions);
if (!$result instanceof \Redis) {
$this->verifyRedisVersionIsSupported();
}
$this->redis->lRem($this->buildKey('entries'), $entryIdentifier, 0);
$this->redis->rPush($this->buildKey('entries'), $entryIdentifier);
foreach ($tags as $tag) {
$this->redis->sAdd($this->buildKey('tag:' . $tag), $entryIdentifier);
$this->redis->sAdd($this->buildKey('tags:' . $entryIdentifier), $tag);
}
$this->redis->exec();
}
public function testsGetMembers()
{
$this->redis->delete('set');
$this->redis->sAdd('set', 'val');
$this->redis->sAdd('set', 'val2');
$this->redis->sAdd('set', 'val3');
$array = array('val', 'val2', 'val3');
$this->assertEquals($array, $this->redis->sGetMembers('set'));
}
/**
* @param $siteUrl
* @param $link
* @param array $data
*/
public function addLink($siteUrl, $link, array $data)
{
if (!$this->siteUrl) {
$this->siteUrl = $siteUrl;
}
$data['link'] = $link;
$this->redis->sAdd($this->getSiteKey($siteUrl), $link);
$this->redis->hMset($this->getPageKey($siteUrl, $link), $data);
}
/**
* Save cache.
*
* @param string $id Cache ID
* @param mixed $data Data to save
* @param int $ttl Time to live in seconds
* @param bool $raw Whether to store the raw value (unused)
*
* @return bool TRUE on success, FALSE on failure
*/
public function save($id, $data, $ttl = 60, $raw = false)
{
if (is_array($data) or is_object($data)) {
if (!$this->_redis->sIsMember('_ci_redis_serialized', $id) && !$this->_redis->sAdd('_ci_redis_serialized', $id)) {
return false;
}
isset($this->_serialized[$id]) or $this->_serialized[$id] = true;
$data = serialize($data);
} elseif (isset($this->_serialized[$id])) {
$this->_serialized[$id] = null;
$this->_redis->sRemove('_ci_redis_serialized', $id);
}
return $this->_redis->set($id, $data, $ttl);
}
/**
* hSave cache
*
* @param string $id Cache ID
* @param mixed $data Data to save
* @param int $ttl Time to live in seconds
* @param bool $raw Whether to store the raw value (unused)
* @return bool TRUE on success, FALSE on failure
*/
public function hsave($id, $key, $data)
{
if (is_array($data) or is_object($data)) {
if (!$this->_redis->sIsMember('_ci_redis_serialized', $id) && !$this->_redis->sAdd('_ci_redis_serialized', $id)) {
return FALSE;
}
isset($this->_serialized[$id]) or $this->_serialized[$id] = TRUE;
$data = base64_encode(serialize($data));
} elseif (isset($this->_serialized[$id])) {
$this->_serialized[$id] = NULL;
$this->_redis->sRemove('_ci_redis_serialized', $id);
}
return $this->_redis->hSet($id, $key, $data);
}
/**
* @param string $key
* @param int $number
* @param int $ttl
* @return int
*/
public function register($key, $number = -1, $ttl = 0)
{
Assertion::notNull(self::$redis, "Redis connection hasn't been set");
Assertion::string($key, "Number key must be a string");
Assertion::notEq($key, "", "Number key must be a non empty string");
Assertion::integer($number, "Temporal numbers must be integers");
Assertion::notEq($number, 0, "Temporal numbers must be integers different from zero");
Assertion::integer($ttl, "Time to live (ttl) must be an integer");
Assertion::true($ttl >= 0, "Time to live (ttl) must be strictly greater than zero");
$key = $this->identifier . "::" . $key;
self::$redis->sAdd($this->identifier, $key);
$ttl == 0 ? self::$redis->set($key, $number) : self::$redis->set($key, $number, $ttl);
return $this->getCurrentNumber();
}
/**
* Save cache
*
* @param string $id Cache ID
* @param mixed $data Data to save
* @param int $ttl Time to live in seconds
* @param bool $raw Whether to store the raw value (unused)
* @return bool TRUE on success, FALSE on failure
*/
public function save($id, $data, $ttl = 60, $raw = FALSE)
{
if (is_array($data) or is_object($data)) {
if (!$this->_redis->sIsMember('_ci_redis_serialized', $id) && !$this->_redis->sAdd('_ci_redis_serialized', $id)) {
return FALSE;
}
isset($this->_serialized[$id]) or $this->_serialized[$id] = TRUE;
$data = serialize($data);
} elseif (isset($this->_serialized[$id])) {
$this->_serialized[$id] = NULL;
$this->_redis->sRemove('_ci_redis_serialized', $id);
}
return $this->_redis->set($id, $data, $ttl);
}
/**
* Save cache
*
* @param string $id Cache ID
* @param mixed $data Data to save
* @param int $ttl Time to live in seconds
* @param bool $raw Whether to store the raw value (unused)
* @return bool TRUE on success, FALSE on failure
*/
public function save($id, $data, $ttl = 60, $raw = FALSE)
{
// //Tim
// if(!$this->is_supported())
// {
// return false;
// }
if (is_array($data) or is_object($data)) {
if (!$this->_redis->sIsMember('_ci_redis_serialized', $id) && !$this->_redis->sAdd('_ci_redis_serialized', $id)) {
return FALSE;
}
isset($this->_serialized[$id]) or $this->_serialized[$id] = TRUE;
$data = serialize($data);
} elseif (isset($this->_serialized[$id])) {
$this->_serialized[$id] = NULL;
$this->_redis->sRemove('_ci_redis_serialized', $id);
}
return $ttl ? $this->_redis->setex($id, $ttl, $data) : $this->_redis->set($id, $data);
}
/**
* Saves data in the cache.
*
* @param string $entryIdentifier An identifier for this specific cache entry
* @param string $data The data to be stored
* @param array $tags Tags to associate with this cache entry. If the backend does not support tags, this option can be ignored.
* @param integer $lifetime Lifetime of this cache entry in seconds. If NULL is specified, the default lifetime is used. "0" means unlimited lifetime.
* @throws \RuntimeException
* @return void
* @api
*/
public function set($entryIdentifier, $data, array $tags = array(), $lifetime = NULL)
{
if ($this->isFrozen()) {
throw new \RuntimeException(sprintf('Cannot add or modify cache entry because the backend of cache "%s" is frozen.', $this->cacheIdentifier), 1323344192);
}
if ($lifetime === NULL) {
$lifetime = $this->defaultLifetime;
}
$setOptions = array();
if ($lifetime > 0) {
$setOptions['ex'] = $lifetime;
}
$this->redis->multi();
$this->redis->set($this->buildKey('entry:' . $entryIdentifier), $data, $setOptions);
$this->redis->rPush($this->buildKey('entries'), $entryIdentifier);
foreach ($tags as $tag) {
$this->redis->sAdd($this->buildKey('tag:' . $tag), $entryIdentifier);
$this->redis->sAdd($this->buildKey('tags:' . $entryIdentifier), $tag);
}
$this->redis->exec();
}
$redis->connect('127.0.0.1');
$redis->select(2);
// HSET
for ($i = 0; $i <= rand(1000, 10000); $i++) {
$incr = $redis->incr('test:incr:hset');
$data = array();
for ($j = 0; $j <= rand(10, 500); $j++) {
$data['field' . $j] = md5($j);
}
$redis->hMset('test:' . $incr . ':hset', $data);
}
// SET
for ($i = 0; $i <= rand(1000, 10000); $i++) {
$incr = $redis->incr('test:incr:set');
for ($j = 0; $j <= rand(10, 100); $j++) {
$redis->sAdd('test:' . $incr . ':set', $j);
}
}
// LIST
for ($i = 0; $i <= rand(1000, 10000); $i++) {
$incr = $redis->incr('test:incr:list');
for ($j = 0; $j <= rand(10, 100); $j++) {
$redis->rPush('test:' . $incr . ':list', $j);
}
}
// STRING
for ($i = 0; $i <= rand(1000, 10000); $i++) {
$incr = $redis->incr('test:incr:string');
for ($j = 0; $j <= rand(10, 100); $j++) {
$redis->set('test:' . $incr . ':string', $j);
}
/**
* Update manufacture
* @param array $data - associative array with data to store
* @return boolean
*/
function update_country($id, $data)
{
$query = $this->db->get_where('countries', array('ID' => $id));
$short_name = $query->row()->short_name;
$start_words_arr = explode("\n", $this->input->post('start_words'));
$error = false;
$this->db->query('BEGIN;SAVEPOINT spcountryUPD;');
$this->db->where('ID', $id);
$this->db->update('countries', $data);
$report = array();
$report['error'] = $this->db->_error_number();
$report['message'] = $this->db->_error_message();
if ($report['error'] == '') {
if (isset($start_words_arr)) {
$new_keys_insert = array();
foreach ($start_words_arr as $row) {
if ($row == "") {
continue;
}
$new_keys_insert[] = trim($row);
}
if (isset($new_keys_insert) and count($new_keys_insert) > 0) {
$redis = new Redis();
$redis->connect('127.0.0.1');
foreach ($new_keys_insert as $insert) {
$redis->sAdd("keys_status:{$short_name}:1", strtolower($insert));
}
$redis->close();
}
}
} else {
$error = true;
}
if ($error) {
$this->db->query('ROLLBACK TO SAVEPOINT spcountryUPD;');
} else {
$this->db->query('COMMIT;');
}
return !$error;
}
private function checkSerializer($mode)
{
$this->redis->delete('key');
$this->assertTrue($this->redis->getOption(Redis::OPT_SERIALIZER) === Redis::SERIALIZER_NONE);
// default
$this->assertTrue($this->redis->setOption(Redis::OPT_SERIALIZER, $mode) === TRUE);
// set ok
$this->assertTrue($this->redis->getOption(Redis::OPT_SERIALIZER) === $mode);
// get ok
// lPush, rPush
$a = array('hello world', 42, TRUE, array('<tag>' => 1729));
$this->redis->delete('key');
$this->redis->lPush('key', $a[0]);
$this->redis->rPush('key', $a[1]);
$this->redis->rPush('key', $a[2]);
$this->redis->rPush('key', $a[3]);
// lGetRange
$this->assertTrue($a === $this->redis->lGetRange('key', 0, -1));
// lGet
$this->assertTrue($a[0] === $this->redis->lGet('key', 0));
$this->assertTrue($a[1] === $this->redis->lGet('key', 1));
$this->assertTrue($a[2] === $this->redis->lGet('key', 2));
$this->assertTrue($a[3] === $this->redis->lGet('key', 3));
// lRemove
$this->assertTrue($this->redis->lRemove('key', $a[3]) === 1);
$this->assertTrue(array_slice($a, 0, 3) === $this->redis->lGetRange('key', 0, -1));
// lSet
$a[0] = array('k' => 'v');
// update
$this->assertTrue(TRUE === $this->redis->lSet('key', 0, $a[0]));
$this->assertTrue($a[0] === $this->redis->lGet('key', 0));
// lInsert
$this->assertTrue($this->redis->lInsert('key', Redis::BEFORE, $a[0], array(1, 2, 3)) === 4);
$this->assertTrue($this->redis->lInsert('key', Redis::AFTER, $a[0], array(4, 5, 6)) === 5);
$a = array(array(1, 2, 3), $a[0], array(4, 5, 6), $a[1], $a[2]);
$this->assertTrue($a === $this->redis->lGetRange('key', 0, -1));
// sAdd
$this->redis->delete('key');
$s = array(1, 'a', array(1, 2, 3), array('k' => 'v'));
$this->assertTrue(1 === $this->redis->sAdd('key', $s[0]));
$this->assertTrue(1 === $this->redis->sAdd('key', $s[1]));
$this->assertTrue(1 === $this->redis->sAdd('key', $s[2]));
$this->assertTrue(1 === $this->redis->sAdd('key', $s[3]));
// variadic sAdd
$this->redis->delete('k');
$this->assertTrue(3 === $this->redis->sAdd('k', 'a', 'b', 'c'));
$this->assertTrue(1 === $this->redis->sAdd('k', 'a', 'b', 'c', 'd'));
// sRemove
$this->assertTrue(1 === $this->redis->sRemove('key', $s[3]));
$this->assertTrue(0 === $this->redis->sRemove('key', $s[3]));
// variadic
$this->redis->delete('k');
$this->redis->sAdd('k', 'a', 'b', 'c', 'd');
$this->assertTrue(2 === $this->redis->sRem('k', 'a', 'd'));
$this->assertTrue(2 === $this->redis->sRem('k', 'b', 'c', 'e'));
$this->assertTrue(FALSE === $this->redis->exists('k'));
// sContains
$this->assertTrue(TRUE === $this->redis->sContains('key', $s[0]));
$this->assertTrue(TRUE === $this->redis->sContains('key', $s[1]));
$this->assertTrue(TRUE === $this->redis->sContains('key', $s[2]));
$this->assertTrue(FALSE === $this->redis->sContains('key', $s[3]));
unset($s[3]);
// sMove
$this->redis->delete('tmp');
$this->redis->sMove('key', 'tmp', $s[0]);
$this->assertTrue(FALSE === $this->redis->sContains('key', $s[0]));
$this->assertTrue(TRUE === $this->redis->sContains('tmp', $s[0]));
unset($s[0]);
// sorted sets
$z = array('z0', array('k' => 'v'), FALSE, NULL);
$this->redis->delete('key');
// zAdd
$this->assertTrue(1 === $this->redis->zAdd('key', 0, $z[0]));
$this->assertTrue(1 === $this->redis->zAdd('key', 1, $z[1]));
$this->assertTrue(1 === $this->redis->zAdd('key', 2, $z[2]));
$this->assertTrue(1 === $this->redis->zAdd('key', 3, $z[3]));
// zDelete
$this->assertTrue(1 === $this->redis->zDelete('key', $z[3]));
$this->assertTrue(0 === $this->redis->zDelete('key', $z[3]));
unset($z[3]);
// check that zDelete doesn't crash with a missing parameter (GitHub issue #102):
$this->assertTrue(FALSE === @$this->redis->zDelete('key'));
// variadic
$this->redis->delete('k');
$this->redis->zAdd('k', 0, 'a');
$this->redis->zAdd('k', 1, 'b');
$this->redis->zAdd('k', 2, 'c');
$this->assertTrue(2 === $this->redis->zDelete('k', 'a', 'c'));
$this->assertTrue(1.0 === $this->redis->zScore('k', 'b'));
$this->assertTrue($this->redis->zRange('k', 0, -1, true) == array('b' => 1.0));
// zRange
$this->assertTrue($z === $this->redis->zRange('key', 0, -1));
// zScore
$this->assertTrue(0.0 === $this->redis->zScore('key', $z[0]));
$this->assertTrue(1.0 === $this->redis->zScore('key', $z[1]));
$this->assertTrue(2.0 === $this->redis->zScore('key', $z[2]));
// zRank
$this->assertTrue(0 === $this->redis->zRank('key', $z[0]));
$this->assertTrue(1 === $this->redis->zRank('key', $z[1]));
$this->assertTrue(2 === $this->redis->zRank('key', $z[2]));
// zRevRank
$this->assertTrue(2 === $this->redis->zRevRank('key', $z[0]));
$this->assertTrue(1 === $this->redis->zRevRank('key', $z[1]));
$this->assertTrue(0 === $this->redis->zRevRank('key', $z[2]));
// zIncrBy
$this->assertTrue(3.0 === $this->redis->zIncrBy('key', 1.0, $z[2]));
$this->assertTrue(3.0 === $this->redis->zScore('key', $z[2]));
$this->assertTrue(5.0 === $this->redis->zIncrBy('key', 2.0, $z[2]));
$this->assertTrue(5.0 === $this->redis->zScore('key', $z[2]));
$this->assertTrue(2.0 === $this->redis->zIncrBy('key', -3.0, $z[2]));
$this->assertTrue(2.0 === $this->redis->zScore('key', $z[2]));
// mset
$a = array('k0' => 1, 'k1' => 42, 'k2' => NULL, 'k3' => FALSE, 'k4' => array('a' => 'b'));
$this->assertTrue(TRUE === $this->redis->mset($a));
foreach ($a as $k => $v) {
$this->assertTrue($this->redis->get($k) === $v);
}
$a = array('k0' => 1, 'k1' => 42, 'k2' => NULL, 'k3' => FALSE, 'k4' => array('a' => 'b'));
// hSet
$this->redis->delete('key');
foreach ($a as $k => $v) {
$this->assertTrue(1 === $this->redis->hSet('key', $k, $v));
}
// hGet
foreach ($a as $k => $v) {
$this->assertTrue($v === $this->redis->hGet('key', $k));
}
// hGetAll
$this->assertTrue($a === $this->redis->hGetAll('key'));
$this->assertTrue(TRUE === $this->redis->hExists('key', 'k0'));
$this->assertTrue(TRUE === $this->redis->hExists('key', 'k1'));
$this->assertTrue(TRUE === $this->redis->hExists('key', 'k2'));
$this->assertTrue(TRUE === $this->redis->hExists('key', 'k3'));
$this->assertTrue(TRUE === $this->redis->hExists('key', 'k4'));
// hMSet
$this->redis->delete('key');
$this->redis->hMSet('key', $a);
foreach ($a as $k => $v) {
$this->assertTrue($v === $this->redis->hGet('key', $k));
}
// hMget
$hmget = $this->redis->hMget('key', array_keys($a));
foreach ($hmget as $k => $v) {
$this->assertTrue($v === $a[$k]);
}
// getMultiple
$this->redis->set('a', NULL);
$this->redis->set('b', FALSE);
$this->redis->set('c', 42);
$this->redis->set('d', array('x' => 'y'));
$this->assertTrue(array(NULL, FALSE, 42, array('x' => 'y')) === $this->redis->getMultiple(array('a', 'b', 'c', 'd')));
// pipeline
$this->sequence(Redis::PIPELINE);
// multi-exec
$this->sequence(Redis::MULTI);
// keys
$this->assertTrue(is_array($this->redis->keys('*')));
// issue #62, hgetall
$this->redis->del('hash1');
$this->redis->hSet('hash1', 'data', 'test 1');
$this->redis->hSet('hash1', 'session_id', 'test 2');
$data = $this->redis->hGetAll('hash1');
$this->assertTrue($data['data'] === 'test 1');
$this->assertTrue($data['session_id'] === 'test 2');
// issue #145, serializer with objects.
$this->redis->set('x', array(new stdClass(), new stdClass()));
$x = $this->redis->get('x');
$this->assertTrue(is_array($x));
$this->assertTrue(is_object($x[0]) && get_class($x[0]) === 'stdClass');
$this->assertTrue(is_object($x[1]) && get_class($x[1]) === 'stdClass');
// revert
$this->assertTrue($this->redis->setOption(Redis::OPT_SERIALIZER, Redis::SERIALIZER_NONE) === TRUE);
// set ok
$this->assertTrue($this->redis->getOption(Redis::OPT_SERIALIZER) === Redis::SERIALIZER_NONE);
// get ok
}
/**
* Add a value to a set
*
* @param string $key
* @param string $value
*/
public function sAdd($key, $value)
{
$this->_redis->sAdd($key, $value);
}
/**
* 将value写入set集合 如果value存在 不写入 返回false
* 如果是有序集合则根据score值更新该元素的顺序
* @param $set string 集合名
* @param $value mixed 值
* @param $stype int 集合类型 0:无序集合 1:有序集和 默认0
* @param $score int 元素排序值
*/
public static function setAdd($set, $value = null, $stype = 0, $score = null)
{
$redis = new \Redis();
$redis->connect(self::_HOST, self::_PORT);
$return = null;
if ($stype && $score !== null) {
$return = $redis->zAdd($set, $score, $value);
} else {
$return = $redis->sAdd($set, $value);
}
$redis->close();
$redis = null;
return $return;
}
public function run()
{
$redis = new Redis();
$redis->pconnect('127.0.0.1');
echo "Run csvParser\n";
$dbconn = pg_connect(DB_CONN_STR) or die('Не могу подключиться к БД: ' . pg_last_error());
//Выберем все не занятые и включенные аккаунты с привязанным прокси
$query = 'SELECT * FROM accounts WHERE status=0 AND busy<>1 AND proxy_ip<>0 ';
$result = pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
$accounts = pg_fetch_all($result);
if (count($accounts) < 1) {
exit;
}
pg_free_result($result);
//Берем рандомный из этого списка
$acc_arr = array_rand($accounts);
//Выбранный аккаунт становится занятым
$query = 'UPDATE accounts SET busy=1 ' . "WHERE id={$accounts[$acc_arr]['id']}";
pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
//Выберем список работающих прокси
$query = 'SELECT * FROM proxys WHERE id = ' . $accounts[$acc_arr]['proxy_ip'];
$result = pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
$proxy_acc = pg_fetch_row($result);
pg_free_result($result);
//Берем случайный из списка
$proxy = "{$proxy_acc[1]}:{$proxy_acc[2]}";
$proxyauth = "{$proxy_acc[3]}:{$proxy_acc[4]}";
//Получаем страну для которой запущен парсер
$query = 'SELECT co."ID",co.full_name country, co.short_name cntr_code FROM countries co, params p ' . 'WHERE co."ID"=p.country_id';
$result = pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
$country = pg_fetch_object($result);
pg_free_result($result);
//Берем первый ключ в статусе "В ожидании" (keys_status = 1)
//Если ключ в статусе "No Keywords" (keys_status = 2) - берем другой
do {
$key = $redis->sPop("keys_status:{$country->cntr_code}:1");
} while ($redis->sIsMember("keys_status:{$country->cntr_code}:2", $key));
//Увеличим количество использования аккаунта
$query = 'UPDATE accounts SET cnt_work=cnt_work+1 ' . "WHERE id={$accounts[$acc_arr]['id']}";
pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
echo "--email={$accounts[$acc_arr]['gm_login']} --pass={$accounts[$acc_arr]['gm_pass']} \n\n --proxy={$proxy} --proxy-auth={$proxyauth} \n\n --key={$key}";
//Получаем время запуска и запускаем скрипт casperjs
$start = microtime(true);
$q = "/usr/local/bin/casperjs {$this->baseDir}googleKeyPlaner.js --proxy=" . $proxy . " --cookies-file={$this->baseDir}cookies/{$key}.txt " . "--proxy-auth=" . $proxyauth . " --ignore-ssl-errors=yes --web-security=no --key='{$key}' " . "--email={$accounts[$acc_arr]['gm_login']} --pass={$accounts[$acc_arr]['gm_pass']} " . "--verphone={$accounts[$acc_arr]['gm_tel']} --country='{$country->country}' > {$this->baseDir}tempFiles/{$key}-exitParser.txt";
exec($q, $ret_arr, $ret_val);
//Если casperjs вернул код выхода 100 - нет keywords для данного ключа. Блокируем ключ
if ($ret_val == 100) {
//Блокируем ключ со статусом "No keywords" (keys_status = 2)
$redis->sAdd("keys_status:{$country->cntr_code}:2", $key);
//Уменьшаем количество использования аккаунта
$query = 'UPDATE accounts SET cnt_work=cnt_work-1 ' . "WHERE id={$accounts[$acc_arr]['id']}";
pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
//Останавливаем процесс
exit;
}
//Скачиваем файл по полученной ссылке для ключа
if (is_file($this->baseDir . 'tempFiles/dnld_' . $key . '.txt')) {
$dnldUrl = file($this->baseDir . 'tempFiles/dnld_' . $key . '.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
echo '=================Start DownLoad=================';
$q = "/usr/local/bin/casperjs {$this->baseDir}downloader.js --proxy=" . $proxy . " --cookies-file={$this->baseDir}cookies/{$key}.txt " . "--proxy-auth=" . $proxyauth . " --ignore-ssl-errors=yes --web-security=no --key='{$key}' " . "--dnldUrl='{$dnldUrl[0]}' > {$this->baseDir}tempFiles/{$key}-exitDownloader.txt";
exec($q, $ret_arr, $ret_val);
}
$work_time = microtime(true) - $start;
//Проверяем наличие скачанного файла для продолжения
if (is_file($this->baseDir . 'tempGrab/stats_' . $key . '.csv')) {
$csvObj = new File_CSV_DataSource();
$csvObj->settings(array('delimiter' => "\t", 'eol' => ''));
$csvObj->load($this->baseDir . 'tempGrab/stats_' . $key . '.csv');
$csvObj->symmetrize(0);
if (!$csvObj->isSymmetric()) {
echo 'Ошибка! Количество заголовков и столбцов в строках не совпадает в файле tempGrab/stats_' . $key . '.csv';
}
//Если файл не пустой
if ($csvObj->countRows() > 0) {
$arrForSHA1 = array(0, 0);
$mainKeyRow = $csvObj->getRow(0);
//Добавляем информацию для заданного ключа
$mainKey['keyword'] = $mainKeyRow[1] != "" ? $mainKeyRow[1] : "";
$mainKey['AMS'] = $mainKeyRow[3] != "" ? $mainKeyRow[3] : 0;
$mainKey['suggested_bid'] = $mainKeyRow[5] != "" ? $mainKeyRow[5] : 0;
$redis->delete("key:{$country->cntr_code}:{$key}");
$redis->hMset("key:{$country->cntr_code}:{$key}", $mainKey);
$arrForSHA1[0] = sha1(serialize($mainKey));
//Добавляем keywords для заданного ключа
$new_words = array();
$keywordRow = array();
for ($i = 1; $i < $csvObj->countRows(); $i++) {
$childKeyRow = $csvObj->getRow($i);
$keywordRow[$i]['keyword'] = $childKeyRow[1] != "" ? $childKeyRow[1] : "";
$keywordRow[$i]['AMS'] = $childKeyRow[3] != "" ? $childKeyRow[3] : 0;
$keywordRow[$i]['suggested_bid'] = $childKeyRow[5] != "" ? $childKeyRow[5] : 0;
//Получаем новые слова для словоря
$new_words = array_merge($new_words, explode(" ", $keywordRow[$i]['keyword']));
}
//Если есть новые слова для словаря
if (count($new_words)) {
//Удаляем дубликаты
$new_words = array_unique($new_words);
//Удаляем стоп-слова
$query = 'SELECT "stop_words" FROM countries ' . 'WHERE "ID"=' . $country->ID;
$result = pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
$st_words_str = pg_fetch_object($result)->stop_words;
pg_free_result($result);
if ($st_words_str) {
$st_words_arr = explode("\n", $st_words_str);
$new_words = array_diff($new_words, $st_words_arr);
}
//Если остались слова - продолжаем
if (count($new_words) > 0) {
//Удаляем однобуквенные слова и слова с запрещенными знаками
foreach ($new_words as $w_key => $value) {
if (!preg_match('|^[a-z]{2}[a-z0-9_-]*$|i', $value)) {
unset($new_words[$w_key]);
continue;
}
}
//Добавляем новые слова в словарь со статусом "В ожидании" (keys_status = 1)
if (count($new_words) > 0) {
$ins_arr = array();
foreach ($new_words as $word) {
if ($redis->exists("key:{$country->cntr_code}:{$word}") || $redis->sIsMember("keys_status:{$country->cntr_code}:2", $word)) {
continue;
}
$redis->sAdd("keys_status:{$country->cntr_code}:1", strtolower(trim($word)));
}
}
}
}
//!!!! ПРОВЕРИТЬ КЛЮЧ В ХЭШ-ТАБЛИЦЕ !!!!!!!!!!!!!!!!!!!!
//Добавляем keywords для ключа
if (count($keywordRow) > 0) {
$arrForSHA1[1] = sha1(serialize($keywordRow));
$i = 1;
$redis->delete($redis->keys("keywords:{$country->cntr_code}:{$key}:*"));
foreach ($keywordRow as $row) {
$redis->hMset("keywords:{$country->cntr_code}:{$key}:{$i}", $row);
$i++;
}
}
$redis->delete("keys_hash:{$country->cntr_code}:{$key}");
$redis->rPush("keys_hash:{$country->cntr_code}:{$key}", $arrForSHA1[0], $arrForSHA1[1]);
} else {
//Файл был пустой - разблокируем ключ (keys_status = 1)
$redis->sAdd("keys_status:{$country->cntr_code}:1", $key);
//Увеличиваем количество ошибок для аккаунта
$query = 'UPDATE accounts SET cnt_fail=cnt_fail+1 ' . "WHERE id={$accounts[$acc_arr]['id']}";
pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
}
//Удаляем скачанный файл
if (is_file($this->baseDir . 'tempGrab/stats_' . $key . '.csv')) {
unlink($this->baseDir . 'tempGrab/stats_' . $key . '.csv');
}
} else {
//Файл не скачался - разблокируем ключ(keys_status = 1)
$redis->sAdd("keys_status:{$country->cntr_code}:1", $key);
//Увеличиваем количество ошибок для аккаунта
$query = 'UPDATE accounts SET cnt_fail=cnt_fail+1 ' . "WHERE id={$accounts[$acc_arr]['id']}";
pg_query($query) or die('Ошибка запроса: ' . pg_last_error());
}
sleep(1);
pg_close($dbconn);
echo "STOP Run csvParser\n";
}
public function testsDiffStore()
{
$this->redis->delete('x');
// set of odd numbers
$this->redis->delete('y');
// set of prime numbers
$this->redis->delete('z');
// set of squares
$this->redis->delete('t');
// set of numbers of the form n^2 - 1
$x = array(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25);
foreach ($x as $i) {
$this->redis->sAdd('x', $i);
}
$y = array(1, 2, 3, 5, 7, 11, 13, 17, 19, 23);
foreach ($y as $i) {
$this->redis->sAdd('y', $i);
}
$z = array(1, 4, 9, 16, 25);
foreach ($z as $i) {
$this->redis->sAdd('z', $i);
}
$t = array(2, 5, 10, 17, 26);
foreach ($t as $i) {
$this->redis->sAdd('t', $i);
}
$count = $this->redis->sDiffStore('k', 'x', 'y');
// x - y
$xy = array_unique(array_diff($x, $y));
$this->assertEquals($count, count($xy));
foreach ($xy as $i) {
$i = (int) $i;
$this->assertTrue($this->redis->sContains('k', $i));
}
$count = $this->redis->sDiffStore('k', 'y', 'z');
// y - z
$yz = array_unique(array_diff($y, $z));
$this->assertEquals($count, count($yz));
foreach ($yz as $i) {
$i = (int) $i;
$this->assertTrue($this->redis->sContains('k', $i));
}
$count = $this->redis->sDiffStore('k', 'z', 't');
// z - t
$zt = array_unique(array_diff($z, $t));
$this->assertEquals($count, count($zt));
foreach ($zt as $i) {
$i = (int) $i;
$this->assertTrue($this->redis->sContains('k', $i));
}
$count = $this->redis->sDiffStore('k', 'x', 'y', 'z');
// x - y - z
$xyz = array_unique(array_diff($x, $y, $z));
$this->assertEquals($count, count($xyz));
foreach ($xyz as $i) {
$i = (int) $i;
$this->assertTrue($this->redis->sContains('k', $i));
}
$this->redis->delete('x');
// x missing now
$count = $this->redis->sDiffStore('k', 'x', 'y', 'z');
// x - y - z
$this->assertTrue($count === 0);
$this->redis->delete('y');
// x and y missing
$count = $this->redis->sDiffStore('k', 'x', 'y', 'z');
// x - y - z
$this->assertTrue($count === 0);
$this->redis->delete('z');
// x, y, and z ALL missing
$count = $this->redis->sDiffStore('k', 'x', 'y', 'z');
// x - y - z
$this->assertTrue($count === 0);
$count = $this->redis->sDiffStore('k');
// diff on nothing...
$this->assertTrue($count === FALSE);
}
public static function addToBlacklist($host)
{
self::$redis->sAdd(self::$prefix . self::BLACKLIST_KEY, $host);
}
