/**
* Destroy Config, Webapp, $_SESSION, $_POST, $_GET, $_REQUEST
*/
public function tearDown()
{
Config::destroyInstance();
PluginRegistrarWebapp::destroyInstance();
PluginRegistrarCrawler::destroyInstance();
if (isset($_SESSION)) {
$this->unsetArray($_SESSION);
}
$this->unsetArray($_POST);
$this->unsetArray($_GET);
$this->unsetArray($_REQUEST);
$this->unsetArray($_SERVER);
$this->unsetArray($_FILES);
Loader::unregister();
$backup_dir = FileDataManager::getBackupPath();
if (file_exists($backup_dir)) {
try {
@exec('cd ' . $backup_dir . '; rm -rf *');
rmdir($backup_dir);
// won't delete if has files
} catch (Exception $e) {
}
}
$data_dir = FileDataManager::getDataPath();
if (file_exists($data_dir . 'compiled_view')) {
try {
@exec('cd ' . $data_dir . '; rm -rf compiled_view');
} catch (Exception $e) {
}
}
parent::tearDown();
}
public function setUp()
{
parent::setUp();
$this->webapp = PluginRegistrarWebapp::getInstance();
$this->crawler = PluginRegistrarCrawler::getInstance();
$this->webapp->registerPlugin('twitter', 'TwitterPlugin');
$this->crawler->registerCrawlerPlugin('TwitterPlugin');
$this->webapp->setActivePlugin('twitter');
$this->logger = Logger::getInstance();
}
public function authControl()
{
if ($this->isAPICall()) {
// If the request comes from an API call, output JSON instead of HTML
$this->setContentType('application/json; charset=UTF-8');
} else {
$this->setContentType('text/html; charset=UTF-8');
$this->setViewTemplate('crawler.run-top.tpl');
echo $this->generateView();
$config = Config::getInstance();
$config->setValue('log_location', false);
//this forces output to just echo to page
$logger = Logger::getInstance();
$logger->close();
}
try {
$logger = Logger::getInstance();
if (isset($_GET['log']) && $_GET['log'] == 'full') {
$logger->setVerbosity(Logger::ALL_MSGS);
echo '<pre style="font-family:Courier;font-size:10px;">';
} else {
$logger->setVerbosity(Logger::USER_MSGS);
$logger->enableHTMLOutput();
}
$crawler_plugin_registrar = PluginRegistrarCrawler::getInstance();
//close session so that it's not locked by long crawl
session_write_close();
$crawler_plugin_registrar->runRegisteredPluginsCrawl();
$logger->close();
} catch (CrawlerLockedException $e) {
if ($this->isAPICall()) {
// Will be caught and handled in ThinkUpController::go()
throw $e;
} else {
// Will appear in the textarea of the HTML page
echo '<td></td><td>' . $e->getMessage() . '</td><td></td>';
}
}
if ($this->isAPICall()) {
echo json_encode((object) array('result' => 'success'));
} else {
$this->setViewTemplate('crawler.run-bottom.tpl');
echo $this->generateView();
}
}
public function control()
{
$output = "";
$authorized = false;
if (isset($this->argc) && $this->argc > 1) {
// check for CLI credentials
$this->content_type = 'text/plain';
$session = new Session();
$username = $this->argv[1];
if ($this->argc > 2) {
$pw = $this->argv[2];
} else {
$pw = getenv('THINKUP_PASSWORD');
}
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$owner = $owner_dao->getByEmail($username);
if ($owner_dao->isOwnerAuthorized($username, $pw)) {
$authorized = true;
Session::completeLogin($owner);
} else {
$output = "ERROR: Incorrect username and password.";
}
} else {
// check user is logged in on the web
if ($this->isLoggedIn()) {
$authorized = true;
} else {
$output = "ERROR: Invalid or missing username and password.";
}
}
if ($authorized) {
$crawler_plugin_registrar = PluginRegistrarCrawler::getInstance();
$crawler_plugin_registrar->runRegisteredPluginsCrawl();
}
return $output;
}
/**
* Provided only for tests that want to kill object in tearDown()
*/
public static function destroyInstance()
{
if (isset(self::$instance)) {
self::$instance = null;
}
}
*/
/**
*
* ThinkUp/webapp/plugins/facebook/controller/facebook.php
*
* Copyright (c) 2009-2015 Gina Trapani
*
* LICENSE:
*
* This file is part of ThinkUp (http://thinkup.com).
*
* ThinkUp is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any
* later version.
*
* ThinkUp is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with ThinkUp. If not, see
* <http://www.gnu.org/licenses/>.
*
* @author Gina Trapani <ginatrapani[at]gmail[dot]com>
* @license http://www.gnu.org/licenses/gpl.html
* @copyright 2009-2015 Gina Trapani
*/
$webapp_plugin_registrar = PluginRegistrarWebapp::getInstance();
$webapp_plugin_registrar->registerPlugin('facebook', 'FacebookPlugin');
$webapp_plugin_registrar->registerPlugin('facebook page', 'FacebookPlugin');
$crawler_plugin_registrar = PluginRegistrarCrawler::getInstance();
$crawler_plugin_registrar->registerCrawlerPlugin('FacebookPlugin');
public function testBitlyCrawl()
{
$builders = $this->buildBitlyData();
$crawler_plugin_registrar = PluginRegistrarCrawler::getInstance();
$config = Config::getInstance();
//use fake Bitly API key
$builders[] = FixtureBuilder::build('options', array('namespace' => OptionDAO::PLUGIN_OPTIONS . '-' . $this->plugin_id, 'option_name' => 'bitly_api_key', 'option_value' => 'dummykey'));
//use fake Bitly login name
$builder[] = FixtureBuilder::build('options', array('namespace' => OptionDAO::PLUGIN_OPTIONS . '-' . $this->plugin_id, 'option_name' => 'bitly_login', 'option_value' => 'bitly123'));
$this->simulateLogin('*****@*****.**', true);
$crawler_plugin_registrar->runRegisteredPluginsCrawl();
$link_dao = DAOFactory::getDAO('LinkDAO');
$link = $link_dao->getLinkById(43);
$this->assertEqual($link->expanded_url, 'http://static.ak.fbcdn.net/rsrc.php/zw/r/ZEKh4ZZQY74.png');
$this->assertEqual($link->title, 'Bitly Test URL');
$this->assertEqual($link->error, '');
$link = $link_dao->getLinkById(42);
$this->assertEqual($link->expanded_url, 'http://bitly.com/a/warning?url=http%3a%2f%2fwww%2ealideas%2ecom%2f&hash=41');
$this->assertEqual($link->error, 'No response from http://bit.ly API');
$link = $link_dao->getLinkById(41);
$this->debug($link->url);
$this->assertEqual($link->expanded_url, 'http://static.ak.fbcdn.net/rsrc.php/zw/r/ZEKh4ZZQY74.png');
$this->assertEqual($link->error, 'No response from http://bit.ly API');
}
public function testOrderOfPluginsWithoutInsightGenerator()
{
$plugin_registrar = new PluginRegistrarCrawler();
$plugin_registrar->registerCrawlerPlugin('TwitterPlugin', true);
$plugin_registrar->registerCrawlerPlugin('FacebookPlugin', true);
$plugin_registrar->registerCrawlerPlugin('FoursquarePlugin', false);
$plugin_registrar->registerCrawlerPlugin('YouTubePlugin', false);
// Order the plugins and check they're in the right order
$plugin_registrar->orderPlugins('crawl');
$plugins = $plugin_registrar->getObjectFunctionCallbacks();
$this->assertNotNull($plugins);
$crawl_plugins = $plugins['crawl'];
$this->assertEqual($crawl_plugins[0][0], 'TwitterPlugin');
$this->assertEqual($crawl_plugins[1][0], 'FacebookPlugin');
$this->assertEqual($crawl_plugins[2][0], 'FoursquarePlugin');
$this->assertEqual($crawl_plugins[3][0], 'YouTubePlugin');
}
public function testGeoEncoderCrawl()
{
$this->simulateLogin('*****@*****.**', true);
$crawler_plugin_registrar = PluginRegistrarCrawler::getInstance();
$crawler_plugin_registrar->runRegisteredPluginsCrawl();
//the crawler closes the log so we have to re-open it
$logger = Logger::getInstance();
$post_dao = DAOFactory::getDAO('PostDAO');
$location_dao = DAOFactory::getDAO('LocationDAO');
// Test 1: Checking Post for Successful Reverse Geoencoding
$this->assertTrue($post_dao->isPostInDB('15645300636', 'twitter'));
$post = $post_dao->getPost('15645300636', 'twitter');
$this->assertEqual($post->is_geo_encoded, 1);
$this->assertEqual($post->geo, '28.602815,77.049136');
$this->assertEqual($post->location, 'Sector 4, New Delhi, Haryana, India');
// Since this is just a post, reply_retweet_distance is 0
$this->assertEqual($post->reply_retweet_distance, 0);
// Test 2: Checking Post for successful Reverse Geoencoding
$post = $post_dao->getPost('15219161227', 'twitter');
$this->assertEqual($post->is_geo_encoded, 1);
$this->assertEqual($post->geo, '28.56213,77.165297');
$this->assertEqual($post->location, 'Vasant Vihar, Munirka, New Delhi, Delhi, India');
// Test: Example of unsuccessful geoencoding resulting out of INVALID_REQUEST.
// NOTE: Not a test case encountered in actual crawl
$post = $post_dao->getPost('15331235880', 'twitter');
$this->assertEqual($post->is_geo_encoded, 5);
$this->assertEqual($post->geo, '28.60abc2815 77.049136');
// Test 1: Checking Post for successful Geoencoding using "place" field
$post = $post_dao->getPost('15052338902', 'twitter');
$this->assertEqual($post->is_geo_encoded, 1);
$this->assertEqual($post->geo, '28.6889398,77.1618859');
$this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi');
$this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India');
// Distance between main post and its reply (Geocoding Process)
$this->assertEqual($post->reply_retweet_distance, 1161);
// Test 2: Checking Post for successful Geoencoding using "place" field
// This post is retrieved from tu_encoded_locations
$post = $post_dao->getPost('14914043658', 'twitter');
$this->assertEqual($post->is_geo_encoded, 1);
$this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi');
$this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India');
// When reply is Not in DB, reply_retweet_distance is -1
$this->assertFalse($post_dao->isPostInDB('999999', 'twitter'));
$this->assertEqual($post->reply_retweet_distance, -1);
// Test 1: Checking Post for successful Geoencoding using "location" field (post had is_geo_encoded set to 3)
$post = $post_dao->getPost('15338041815', 'twitter');
$this->assertEqual($post->geo, '19.017656,72.856178');
$this->assertEqual($post->place, NULL);
$this->assertEqual($post->location, 'Mumbai, Maharashtra, India');
$this->assertEqual($post->is_geo_encoded, 1);
// Test 2: Checking Post for successful Geoencoding using "location" field
$post = $post_dao->getPost('15344199472', 'twitter');
$this->assertEqual($post->location, 'New Delhi, Delhi, India');
$this->assertEqual($post->is_geo_encoded, 1);
// Distance between Post and Retweet (Geocoding Process)
$this->assertEqual($post->reply_retweet_distance, 18);
// When all three fields are filled, <geo> is given the most preference
$post = $post_dao->getPost('11259110570', 'twitter');
$this->assertEqual($post->geo, '28.56213,77.165297');
$this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi');
$this->assertEqual($post->location, 'Vasant Vihar, Munirka, New Delhi, Delhi, India');
$this->assertEqual($post->is_geo_encoded, 1);
// Distance between reply and post (Reverse Geocoding Process)
$this->assertEqual($post->reply_retweet_distance, 14);
// When only place and location are filled, <place> is given preference
$post = $post_dao->getPost('15052338902', 'twitter');
$this->assertEqual($post->geo, '28.6889398,77.1618859');
$this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi');
$this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India');
$this->assertEqual($post->is_geo_encoded, 1);
// Unsuccessful Geoencoding due to place field
// NOTE: Not a test case encountered in real crawl
$post = $post_dao->getPost('14913946516', 'twitter');
$this->assertEqual($post->geo, NULL);
$this->assertEqual($post->place, 'abc');
$this->assertEqual($post->location, 'New Delhi');
$this->assertEqual($post->is_geo_encoded, 2);
$this->assertEqual($post->reply_retweet_distance, 0);
//Unsuccessful Geoencoding due to location field
$post = $post_dao->getPost('15268690400', 'twitter');
$this->assertEqual($post->geo, NULL);
$this->assertEqual($post->place, NULL);
$this->assertEqual($post->location, 'abc');
$this->assertEqual($post->is_geo_encoded, 2);
//Unsuccessful Geoencoding due to location field resulting in INVALID_REQUEST
// Removing this test due to inconsistent behavior between testing environments for non-UTF8 chars and errors
// like:
// General error: 1366 Incorrect string value: '\xDC' for column 'location' at row 1
//$post = $post_dao->getPost('15244973830', 'twitter');
//$this->assertEqual($post->location, 'Ü');
//$this->assertEqual($post->is_geo_encoded, 5);
//Unsuccessful Geoencoding due to all three fields being empty
$post = $post_dao->getPost('15435434230', 'twitter');
$this->assertEqual($post->geo, NULL);
$this->assertEqual($post->place, NULL);
$this->assertEqual($post->location, NULL);
$this->assertEqual($post->is_geo_encoded, 6);
//Reverse Geoencoding when latitude and longitude are found in location field instead of geo field
$post = $post_dao->getPost('13212618909', 'twitter');
$this->assertEqual($post->geo, '40.681839,-73.983734');
$this->assertEqual($post->place, NULL);
$this->assertEqual($post->location, 'Boerum Hill, Brooklyn, NY, USA');
$this->assertEqual($post->is_geo_encoded, 1);
// Retweet Distance in case of Reverse Geocoding Process
$this->assertEqual($post->reply_retweet_distance, 11760);
//Unsuccessful Geoencoding due to REQUEST_DENIED
$post = $post_dao->getPost('12259110570', 'twitter');
$this->assertEqual($post->place, 'request_denied');
$this->assertEqual($post->is_geo_encoded, 4);
//Unsuccessful Geoencoding due to OVER_QUERY_LIMIT
$post = $post_dao->getPost('13259110570', 'twitter');
$this->assertEqual($post->place, 'over_query_limit');
$this->assertEqual($post->is_geo_encoded, 3);
//After reaching OVER_QUERY_LIMIT, next posts are not geoencoded
$post = $post_dao->getPost('15645301636', 'twitter');
$this->assertEqual($post->is_geo_encoded, 0);
$post = $post_dao->getPost('11331235880', 'twitter');
$this->assertEqual($post->is_geo_encoded, 0);
// Check up filling of tu_encoded_locations table
$locations = $location_dao->getAllLocations();
$this->assertEqual(count($locations), 6);
$this->assertEqual($locations[0]['short_name'], "28.602815 77.049136");
$this->assertEqual($locations[2]['short_name'], "Mumbai");
$this->assertEqual($locations[5]['short_name'], "40.681839 -73.983734");
}
public function testCrawlUpgrading()
{
// up app version
$config = Config::getInstance();
$init_db_version = $config->getValue('THINKUP_VERSION');
$config->setValue('THINKUP_VERSION', $config->getValue('THINKUP_VERSION') + 10);
//set a high version num
$builders = $this->buildData();
$crawler_plugin_registrar = PluginRegistrarCrawler::getInstance();
$crawler_plugin_registrar->registerPlugin('hellothinkup', 'HelloThinkUpPlugin');
$crawler_plugin_registrar->registerCrawlerPlugin('HelloThinkUpPlugin');
$this->simulateLogin('*****@*****.**', true);
$this->expectException(new InstallerException('ThinkUp needs a database migration, so we are unable to run the crawler.'));
$crawler_plugin_registrar->runRegisteredPluginsCrawl();
// reset version
$config->setValue('THINKUP_VERSION', $init_db_version);
}
