/**
* Make sure the data is actually stored in the databse
*/
public function testCreateBucketDataStored()
{
$bucket_data = array('_id' => 'test-bucket', 'description' => 'Test Bucket');
$collection = $this->mongo->selectCollection(Bucket::$collection);
$this->assertEquals(0, $collection->count());
$bucket = new Bucket($bucket_data);
$bucket->save();
$this->assertEquals(1, $collection->count());
$this->assertEquals(array('description' => 'Test Bucket'), $collection->findOne(array(), array('_id' => 0, 'description' => 1)));
}
public function after()
{
// All Applications
$this->set('buckets', BucketModel::find());
// Applications for this user
$this->set('user_buckets', $this->user->getBuckets());
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$dialog = $this->getHelperSet()->get('dialog');
$hoard_host = $dialog->ask($output, 'Hoard Host: ', 'hoard.dev');
$request_count = $dialog->ask($output, 'Number of Events: ', 10000);
// Assert Fake Bucket
$id = '51cd6e83c2cea';
$alias = 'demo-bucket';
$data = array('_id' => $id, 'alias' => array($alias), 'description' => 'Demo Bucket', 'roles' => array('all' => 'owner'), 'created' => new \MongoDate(), 'updated' => new \MongoDate());
$bucket = Bucket::create($data);
// Events
$events = array('test1', 'test2', 'test3', 'test4', 'test5');
// Get Faker Factory
$faker = \Faker\Factory::create();
// Now pump data in
$run = true;
$count = 0;
while ($run && $count < $request_count) {
$event = $events[array_rand($events)];
$post = json_encode(array('v' => 1, 'b' => $alias, 'e' => $event, 'd' => array('name' => $faker->name, 'gender' => rand(0, 1) === 1 ? 'male' : 'female', 'country' => $faker->countryCode, 'response_time' => rand(1, 1200))));
$ch = curl_init('http://' . $hoard_host . '/api/track?apikey=' . $this->apikey);
curl_setopt_array($ch, array(CURLOPT_RETURNTRANSFER => true, CURLOPT_POST => true, CURLOPT_POSTFIELDS => $post));
$response = curl_exec($ch);
$count++;
echo "\rCount: " . str_pad(number_format($count), 10, ' ', STR_PAD_RIGHT) . ' ' . $response . ' ';
usleep(rand(100, 10000));
}
}
public function req_post()
{
if ($this->app->request->get('action') === 'create_bucket') {
$name = $this->app->request->get('bucket_name');
$alias = strtolower($name);
$alias = str_replace(array(' ', '_'), '-', $alias);
$pattern = Bucket::$regex_id;
// No name is specified
if (!$alias) {
$this->alert('You need to specify a name');
} elseif (!preg_match($pattern, $alias)) {
$this->alert('Invalid Alias. Please match <strong>' . $pattern . '</strong>');
} elseif (Bucket::exists($alias)) {
$this->alert('Bucket name must be unique across cluster');
} else {
$data = array('alias' => array($alias), 'description' => $name, 'roles' => array((string) $this->app->auth->id => 'owner'), 'created' => new \MongoDate(), 'updated' => new \MongoDate());
$bucket = Bucket::create($data);
if ($bucket) {
$this->alert('Your app was created. <a href="/bucket/' . $alias . '">' . $name . '</a>');
} else {
$this->alert('There was a problem creating your bucket', 'error');
}
}
}
// Do normal GET request
return $this->req_get();
}
public function before()
{
// Require Authentication
if (!$this->isLoggedIn()) {
header('Location: /login/');
exit;
}
// Get app key
$this->id = isset($this->uri[1]) ? $this->uri[1] : '';
if (!$this->id) {
header('Location: /');
exit;
}
$this->bucket = BucketModel::findById($this->id);
if ($this->bucket === null) {
// LEGACY: Check mongoid from legacy systems
$this->bucket = BucketModel::findById(new \MongoId($this->id));
if ($this->bucket === null) {
header('Location: /');
exit;
}
}
// Check if legacy
if ($this->bucket->legacy) {
$this->alert('This bucket is running in legacy mode. Please upgrade!');
}
// Check action
$app_action = isset($this->uri[2]) ? $this->uri[2] : '';
$collection = $this->app->mongo->selectCollection(BucketModel::$collection);
switch ($app_action) {
case 'save':
$alias_string = $this->app->request->get('alias');
$explode = explode(',', $alias_string);
$aliases = array();
foreach ($explode as $alias) {
$alias = trim($alias);
if (preg_match(BucketModel::$regex_id, $alias)) {
$aliases[] = $alias;
}
}
$this->bucket->alias = $aliases;
$this->bucket->description = $this->app->request->get('description');
$this->bucket->save();
break;
case 'delete':
$collection->remove(array('_id' => $this->bucket->id));
$this->app->mongo->selectCollection($this->bucket->event_collection)->drop();
header('Location: /');
exit;
break;
case 'empty':
$this->app->mongo->selectCollection($this->bucket->event_collection)->drop();
header('Location: /bucket/' . $this->id);
exit;
break;
}
}
public function req_get()
{
header('Content-Type: text/plain');
$req = $this->app->request;
$params = array_merge($req->query->all(), $req->request->all());
$format = 'json';
$dataload = $req->get('data') ? json_decode(urldecode($req->get('data')), true) : array();
$payload = $req->get('payload') ? json_decode(urldecode($req->get('payload')), true) : array();
$data = array_merge($dataload, $payload);
// Event is required
$event = $req->get('event') ?: (isset($data['event']) ? $data['event'] : (!empty($this->uri[1]) ? $this->uri[1] : false));
if (!$event) {
echo '500 No Event Specified';
exit;
}
// Get Bucket Instance
if ($req->get('bucket')) {
$bucket_id = $req->get('bucket');
} else {
$bucket_id = array_key_exists('appkey', $params) ? $params['appkey'] : (array_key_exists('appkey', $data) ? $data['appkey'] : false);
}
$bucket_id = trim($bucket_id);
if (!$bucket_id) {
echo '500 No Bucket ID Specified';
exit;
}
$bucket = Bucket::findById($bucket_id);
if (!$bucket) {
echo '500 Invalid Bucket ID';
exit;
}
// Normalize Data
unset($data['event']);
unset($data['appkey']);
unset($data['bucket']);
unset($data['sig'], $data['hash']);
// Append to data
$insert = array();
$insert['t'] = new \MongoDate();
$insert['e'] = $event;
$insert['d'] = $data;
// Save Data to log
try {
$collection = $this->app->mongo->selectCollection($bucket->event_collection);
$collection->insert($insert);
echo $insert['_id'];
exit;
} catch (MongoConnectionException $e) {
echo '503 Database Exception';
exit;
}
exit;
}
public function exec()
{
$debug = $this->app->request->get('debug') ? true : false;
// Get JSON Body
if ($this->app->request->getMethod() === 'POST') {
$postBody = file_get_contents('php://input');
if (!$postBody) {
return $this->error('No data', 400);
}
$payload_data = json_decode($postBody, true);
} else {
$payload_data = json_decode($this->app->request->get('payload') ?: '', true);
}
// Convert into standardized payload
$payload = new Payload($payload_data);
if (!$payload->isVersionSupported()) {
return $this->error('Payload version ' . $payload->version . ' not supported');
}
// Verify Event Credentials
if (!$payload->event) {
return $this->error('No Event specified', 400);
}
// Verify Bucket Credentials
if (!$payload->bucket) {
return $this->error('No Bucket specified', 400);
}
$bucket_id = $payload->bucket;
$bucket = Bucket::findById($bucket_id);
if ($bucket === null) {
return $this->error('Invalid Bucket', 404);
}
// Save Event
$insert = array();
$insert['t'] = $payload->time;
$insert['e'] = $payload->event;
$insert['d'] = $payload->data;
// Save Data to log
$id = null;
try {
$collection = $this->app->mongo->selectCollection($bucket->event_collection);
$collection->insert($insert);
$id = (string) $insert['_id'];
} catch (\Exception $e) {
// TODO: Queue event for later processing
return $this->error('Database Exception', 503);
}
// Output Results
$output = array('data' => array('id' => $id));
if ($debug) {
$output['debug'] = array('payload' => $payload->asArray());
}
return $output;
}
/**
* Since buckets can be aliased, we must test this
*/
public function testBucketAliasPayload()
{
// Create Bucket
$bucket = Bucket::create(array('_id' => '51d077a88dff0', 'alias' => array('test-bucket-2')));
// Create payload
$payload = array('v' => 1, 'b' => $bucket->alias[0], 'e' => 'test-event', 'd' => array('test1' => 1, 'test2' => 2));
// Make request
$response = $this->makeApiRequest('GET', '/api/track?payload=' . urlencode(json_encode($payload)));
$data = json_decode($response->getContent(), true);
$this->assertArrayHasKey('id', $data['data']);
// Update alias and do it again
$bucket->alias = array('test-bucket-3');
$bucket->save();
$response = $this->makeApiRequest('GET', '/api/track?payload=' . urlencode(json_encode($payload)));
$data = json_decode($response->getContent(), true);
$this->assertEquals(404, $data['error']['code']);
$this->assertEquals('Invalid Bucket', $data['error']['message']);
// Musical bucket alias's!
$bucket->alias = array('test-bucket-2');
$bucket->save();
$response = $this->makeApiRequest('GET', '/api/track?payload=' . urlencode(json_encode($payload)));
$data = json_decode($response->getContent(), true);
$this->assertArrayHasKey('id', $data['data']);
}
public function req_get()
{
// Get bucket
$this->bucket_id = $this->app->request->get('bucket');
if (!$this->bucket_id) {
return $this->jsonError('No bucket specified');
}
$this->bucket = Bucket::findById($this->bucket_id);
if (!$this->bucket) {
return $this->jsonError('Invalid Bucket ID', 404);
}
$this->collection = $this->app->mongo->selectCollection($this->bucket->event_collection);
// Event filtering
$this->event = $this->app->request->get('event');
// Custom Queries
$query = $this->app->request->get('query');
if (is_string($query)) {
$this->query = json_decode($query, true) ?: null;
}
// Verify input (TODO: Stronger validation and conversion)
$period = $this->app->request->get('period') ?: false;
$now = time();
$default_time_gap = 1800;
$default_time_step = 60;
// Per day
if ($period === 'month' || $period === 18144000) {
$default_time_gap = 18144000;
$default_time_step = 86400;
} elseif ($period === 'week' || $period === 604800) {
$default_time_gap = 604800;
$default_time_step = 86400;
} elseif ($period === 'day' || $period === '86400') {
$default_time_gap = 86400;
$default_time_step = 3600;
} elseif ($period === 'hour' || $period == '3600') {
$default_time_gap = 3600;
$default_time_step = 60;
} elseif ($period === 'minute' || $period == '60') {
$default_time_gap = 60;
$default_time_step = 1;
} elseif ((int) $period) {
$default_time_gap = $period;
$default_time_step = $default_time_gap / 30;
}
// Build time groups
$this->time_step = (int) $this->app->request->get('step') ?: $default_time_step;
$this->time_gap = $default_time_gap;
// Make sure there aren't too many steps (possiblity of crashing stats engine)
if ($this->time_gap / $this->time_step > 300) {
$this->time_step = $this->time_gap / 300;
}
// Create time ranges
$this->time_start = $now - $this->time_gap - ($now - $this->time_gap) % $this->time_step;
$this->time_end = $now - $now % $this->time_step + $this->time_step;
// Build Query
$results = array();
$query = array();
if ($this->event) {
$query['e'] = $this->event;
}
if ($this->query) {
foreach ($this->query as $key => $val) {
$query['d.' . $key] = $val;
}
}
// Function
$func = '$sum';
$func_inc = 1;
$func_override = $this->app->request->get('func');
if ($func_override) {
list($func_name, $func_var) = explode(':', $func_override);
if ($func_name === 'avg') {
$func = '$avg';
$func_inc = '$d.' . $func_var;
}
}
// Loop over times
for ($time = $this->time_start; $time < $this->time_end; $time += $this->time_step) {
$query['t'] = array('$gte' => new \MongoDate($time), '$lte' => new \MongoDate($time + $this->time_step));
$op = array(array('$match' => $query), array('$group' => array('_id' => '$e', 'v' => array($func => $func_inc))));
$aggregate = $this->app->mongo->command(array('aggregate' => $this->collection->getName(), 'pipeline' => $op));
$count = 0;
$result_arr = array();
foreach ($aggregate['result'] as $result) {
$count += $result['v'];
$result_arr[$result['_id']] = $result['v'];
}
// Calculate Average
if ($func === '$avg') {
if ($result_arr) {
$count = round($count / count($result_arr), 2);
}
}
$results[] = array('range' => array($time, $time + $this->time_step), 'count' => $count, 'events' => $result_arr);
}
// Output Results
return $this->json($results, 200, array('range' => array($this->time_start, $this->time_end), 'step' => $this->time_step));
}
public function req_get()
{
header('Content-Type: text/plain');
$params = $_GET + $_POST;
// Retrict Access to logged in users
// if ( ! Auth::$id)
// {
// echo '{"error":"Authentication Required"}';
// exit;
// }
// App Key Required, Secret too in future
$bucket_id = $this->app->request->get('bucket') ?: $this->uri[1];
if (empty($bucket_id)) {
return $this->jsonError('Bucket ID is Required', 400);
}
$bucket = BucketModel::findById($bucket_id);
if (!$bucket) {
return $this->jsonError('Invalid Bucket ID', 404);
}
// Vars
//$event = isset($this->uri[1]) ? $this->uri[1] : false;
$event = $this->app->request->get('event') ?: '';
$limit = isset($params['limit']) ? (int) $params['limit'] : 10;
if ($limit < 1) {
$limit = 10;
}
// Where
$where = array();
if ($event) {
$where['e'] = $event;
}
/*
if ($bucket) {
$where['appkey'] = $bucket;
} else {
$app_keys = array();
foreach (Auth::$buckets as $k => $app) {
$app_keys[] = $app['appkey'];
}
$where['appkey'] = array('$in' => $app_keys);
}
*/
if (!empty($params['query'])) {
$json = $this->json2array($params['query'], true);
if (is_scalar($json) || !is_array($json)) {
$json = array('$e' => $params['query']);
}
foreach ($json as $k => $v) {
// Specials
if ($k === '$e') {
$where['e'] = $v;
continue;
}
if (is_array($v)) {
foreach ($v as $_k1 => $_v1) {
if ($_k1 === '$regex') {
$v[$_k1] = new MongoRegex($_v1);
}
}
}
$where['d.' . $k] = $v;
}
}
// print_r($where); exit;
if (isset($where['_id'])) {
$where['_id'] = new MongoId($where['_id']);
}
// Fields
$fields = array();
if (!empty($params['fields'])) {
$explode = explode(',', $params['fields']);
foreach ($explode as $field) {
$field = trim($field);
$fields['d.' . $field] = 1;
}
}
if ($fields) {
$fields['t'] = 1;
$fields['e'] = 1;
}
// Sort
$sort = array();
if (!empty($params['sort'])) {
$json = $this->json2array($params['sort'], true);
if (!$json) {
preg_match('/^([^:]+)(:([\\-]*1))?$/', $params['sort'], $matches);
if (isset($matches[1])) {
$order = isset($matches[3]) && $matches[3] === '-1' ? -1 : 1;
$json = array($matches[1] => $order);
}
}
foreach ($json as $k => $v) {
if ($k === '$time') {
$sort['t'] = $v;
} else {
$sort["d." . $k] = $v;
}
}
}
if (!$sort) {
$sort['t'] = -1;
}
// print_r($sort);
// Find Data
// Save Data to log
try {
$collection = $this->app->mongo->selectCollection($bucket->event_collection);
try {
$cursor = $collection->find($where, $fields)->limit($limit);
if ($sort) {
$cursor->sort($sort);
}
$data = array();
foreach ($cursor as $row) {
$row['_id'] = (string) $row['_id'];
// $row['date'] = (array) $row['t'];
$data[] = $row;
}
echo json_encode($data);
} catch (MongoCursorException $e) {
echo '{"error":"Cursor Exception"}';
exit;
}
exit;
} catch (MongoConnectionException $e) {
echo '{"error":"Connection Exception"}';
exit;
}
// Output
exit;
}
public function exec()
{
$request = $this->app->request;
// Bucket ID is required
$bucket_id = $request->get('bucket');
if (empty($bucket_id)) {
return $this->error('Bucket ID is Required', 400);
}
$bucket = Bucket::findById($bucket_id);
if (!$bucket) {
return $this->error('Invalid Bucket ID', 404);
}
// Grab request params
$event = $request->get('event') ?: '';
$limit = (int) $request->get('limit') ?: 10;
// Where
$where_param = $request->get('where');
$where = array();
if ($event) {
$where['e'] = $event;
}
if (!empty($where_param)) {
$json = json_decode($where_param, true);
if (is_scalar($json) || !is_array($json)) {
$json = array('$e' => $where_param);
}
foreach ($json as $k => $v) {
// Specials
if ($k === '$e') {
$where['e'] = $v;
continue;
}
if (is_array($v)) {
foreach ($v as $_k1 => $_v1) {
if ($_k1 === '$regex') {
$v[$_k1] = new MongoRegex($_v1);
}
}
}
$where['d.' . $k] = $v;
}
}
// Fields
$fields_param = $request->get('fields');
$fields = array();
if (!empty($fields_param)) {
$explode = explode(',', $fields_param);
foreach ($explode as $field) {
$field = trim($field);
$fields['d.' . $field] = 1;
}
}
if ($fields) {
$fields['t'] = 1;
$fields['e'] = 1;
}
// Sorting
$sort_param = $request->get('sort');
$sort = array();
if (!empty($sort_param)) {
$json = json_decode($sort_param, true);
if (!$json) {
preg_match('/^([^:]+)(:([\\-]*1))?$/', $params['sort'], $matches);
if (isset($matches[1])) {
$order = isset($matches[3]) && $matches[3] === '-1' ? -1 : 1;
$json = array($matches[1] => $order);
}
}
foreach ($json as $k => $v) {
if ($k === '$time') {
$sort['t'] = $v;
} else {
$sort['d.' . $k] = $v;
}
}
}
if (!$sort) {
$sort['t'] = -1;
}
// Grab from database
try {
$collection = $this->app->mongo->selectCollection($bucket->event_collection);
try {
$cursor = $collection->find($where, $fields)->limit($limit);
if ($sort) {
$cursor->sort($sort);
}
$data = array();
foreach ($cursor as $row) {
$item = array('id' => (string) $row['_id'], 'event' => $row['e'], 'data' => $row['d'], 'time' => [$row['t']->sec, $row['t']->usec]);
$data[] = $item;
}
} catch (MongoCursorException $e) {
return $this->error('Database Write Error', 503);
}
} catch (MongoConnectionException $e) {
return $this->error('Database Connection Error', 503);
}
// Output Results
return array('data' => $data, 'meta' => array('where' => $where, 'fields' => $fields, 'limit' => $limit, 'sort' => $sort));
}
/**
* Create test bucket
*/
public function createTestBucket()
{
$bucket = Bucket::create(array('_id' => '51d077a88dff0', 'alias' => array('test-bucket')));
return $bucket;
}