public function setUp() { parent::setUp(); $this->webapp = PluginRegistrarWebapp::getInstance(); $this->crawler = PluginRegistrarCrawler::getInstance(); $this->webapp->registerPlugin('twitter', 'TwitterPlugin'); $this->crawler->registerCrawlerPlugin('TwitterPlugin'); $this->webapp->setActivePlugin('twitter'); $this->logger = Logger::getInstance(); }
/** * Initialize Config and Webapp objects, clear $_SESSION, $_POST, $_GET, $_REQUEST */ public function setUp() { parent::setUp(); Loader::register(array(THINKUP_ROOT_PATH . 'tests/', THINKUP_ROOT_PATH . 'tests/classes/', THINKUP_ROOT_PATH . 'tests/fixtures/')); $config = Config::getInstance(); //disable caching for tests $config->setValue('cache_pages', false); //tests assume profiling is off $config->setValue('enable_profiler', false); if ($config->getValue('timezone')) { date_default_timezone_set($config->getValue('timezone')); } $webapp_plugin_registrar = PluginRegistrarWebapp::getInstance(); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $this->DEBUG = getenv('TEST_DEBUG') !== false ? true : false; self::isTestEnvironmentReady(); }
public function authControl() { if ($this->isAPICall()) { // If the request comes from an API call, output JSON instead of HTML $this->setContentType('application/json; charset=UTF-8'); } else { $this->setContentType('text/html; charset=UTF-8'); $this->setViewTemplate('crawler.run-top.tpl'); echo $this->generateView(); $config = Config::getInstance(); $config->setValue('log_location', false); //this forces output to just echo to page $logger = Logger::getInstance(); $logger->close(); } try { $logger = Logger::getInstance(); if (isset($_GET['log']) && $_GET['log'] == 'full') { $logger->setVerbosity(Logger::ALL_MSGS); echo '<pre style="font-family:Courier;font-size:10px;">'; } else { $logger->setVerbosity(Logger::USER_MSGS); $logger->enableHTMLOutput(); } $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); //close session so that it's not locked by long crawl session_write_close(); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); $logger->close(); } catch (CrawlerLockedException $e) { if ($this->isAPICall()) { // Will be caught and handled in ThinkUpController::go() throw $e; } else { // Will appear in the textarea of the HTML page echo '<td></td><td>' . $e->getMessage() . '</td><td></td>'; } } if ($this->isAPICall()) { echo json_encode((object) array('result' => 'success')); } else { $this->setViewTemplate('crawler.run-bottom.tpl'); echo $this->generateView(); } }
/** * Initialize Config and Webapp objects, clear $_SESSION, $_POST, $_GET, $_REQUEST */ public function setUp() { parent::setUp(); Loader::register(array(THINKUP_ROOT_PATH . 'tests/', THINKUP_ROOT_PATH . 'tests/classes/', THINKUP_ROOT_PATH . 'tests/fixtures/')); $config = Config::getInstance(); //disable caching for tests $config->setValue('cache_pages', false); //tests assume profiling is off $config->setValue('enable_profiler', false); if ($config->getValue('timezone')) { date_default_timezone_set($config->getValue('timezone')); } //tests assume no redirect to ThinkUp LLC if ($config->getValue('thinkupllc_endpoint') != null) { $config->setValue('thinkupllc_endpoint', null); } $webapp_plugin_registrar = PluginRegistrarWebapp::getInstance(); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $this->DEBUG = getenv('TEST_DEBUG') !== false ? true : false; ini_set('session.use_cookies', 0); session_cache_limiter(''); self::isTestEnvironmentReady(); }
public function control() { $output = ""; $authorized = false; if (isset($this->argc) && $this->argc > 1) { // check for CLI credentials $this->content_type = 'text/plain'; $session = new Session(); $username = $this->argv[1]; if ($this->argc > 2) { $pw = $this->argv[2]; } else { $pw = getenv('THINKUP_PASSWORD'); } $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail($username); if ($owner_dao->isOwnerAuthorized($username, $pw)) { $authorized = true; Session::completeLogin($owner); } else { $output = "ERROR: Incorrect username and password."; } } else { // check user is logged in on the web if ($this->isLoggedIn()) { $authorized = true; } else { $output = "ERROR: Invalid or missing username and password."; } } if ($authorized) { $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); } return $output; }
*/ /** * * ThinkUp/webapp/plugins/facebook/controller/facebook.php * * Copyright (c) 2009-2015 Gina Trapani * * LICENSE: * * This file is part of ThinkUp (http://thinkup.com). * * ThinkUp is free software: you can redistribute it and/or modify it under the terms of the GNU General Public * License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any * later version. * * ThinkUp is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. * * You should have received a copy of the GNU General Public License along with ThinkUp. If not, see * <http://www.gnu.org/licenses/>. * * @author Gina Trapani <ginatrapani[at]gmail[dot]com> * @license http://www.gnu.org/licenses/gpl.html * @copyright 2009-2015 Gina Trapani */ $webapp_plugin_registrar = PluginRegistrarWebapp::getInstance(); $webapp_plugin_registrar->registerPlugin('facebook', 'FacebookPlugin'); $webapp_plugin_registrar->registerPlugin('facebook page', 'FacebookPlugin'); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $crawler_plugin_registrar->registerCrawlerPlugin('FacebookPlugin');
public function testBitlyCrawl() { $builders = $this->buildBitlyData(); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $config = Config::getInstance(); //use fake Bitly API key $builders[] = FixtureBuilder::build('options', array('namespace' => OptionDAO::PLUGIN_OPTIONS . '-' . $this->plugin_id, 'option_name' => 'bitly_api_key', 'option_value' => 'dummykey')); //use fake Bitly login name $builder[] = FixtureBuilder::build('options', array('namespace' => OptionDAO::PLUGIN_OPTIONS . '-' . $this->plugin_id, 'option_name' => 'bitly_login', 'option_value' => 'bitly123')); $this->simulateLogin('*****@*****.**', true); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); $link_dao = DAOFactory::getDAO('LinkDAO'); $link = $link_dao->getLinkById(43); $this->assertEqual($link->expanded_url, 'http://static.ak.fbcdn.net/rsrc.php/zw/r/ZEKh4ZZQY74.png'); $this->assertEqual($link->title, 'Bitly Test URL'); $this->assertEqual($link->error, ''); $link = $link_dao->getLinkById(42); $this->assertEqual($link->expanded_url, 'http://bitly.com/a/warning?url=http%3a%2f%2fwww%2ealideas%2ecom%2f&hash=41'); $this->assertEqual($link->error, 'No response from http://bit.ly API'); $link = $link_dao->getLinkById(41); $this->debug($link->url); $this->assertEqual($link->expanded_url, 'http://static.ak.fbcdn.net/rsrc.php/zw/r/ZEKh4ZZQY74.png'); $this->assertEqual($link->error, 'No response from http://bit.ly API'); }
public function testGeoEncoderCrawl() { $this->simulateLogin('*****@*****.**', true); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); //the crawler closes the log so we have to re-open it $logger = Logger::getInstance(); $post_dao = DAOFactory::getDAO('PostDAO'); $location_dao = DAOFactory::getDAO('LocationDAO'); // Test 1: Checking Post for Successful Reverse Geoencoding $this->assertTrue($post_dao->isPostInDB('15645300636', 'twitter')); $post = $post_dao->getPost('15645300636', 'twitter'); $this->assertEqual($post->is_geo_encoded, 1); $this->assertEqual($post->geo, '28.602815,77.049136'); $this->assertEqual($post->location, 'Sector 4, New Delhi, Haryana, India'); // Since this is just a post, reply_retweet_distance is 0 $this->assertEqual($post->reply_retweet_distance, 0); // Test 2: Checking Post for successful Reverse Geoencoding $post = $post_dao->getPost('15219161227', 'twitter'); $this->assertEqual($post->is_geo_encoded, 1); $this->assertEqual($post->geo, '28.56213,77.165297'); $this->assertEqual($post->location, 'Vasant Vihar, Munirka, New Delhi, Delhi, India'); // Test: Example of unsuccessful geoencoding resulting out of INVALID_REQUEST. // NOTE: Not a test case encountered in actual crawl $post = $post_dao->getPost('15331235880', 'twitter'); $this->assertEqual($post->is_geo_encoded, 5); $this->assertEqual($post->geo, '28.60abc2815 77.049136'); // Test 1: Checking Post for successful Geoencoding using "place" field $post = $post_dao->getPost('15052338902', 'twitter'); $this->assertEqual($post->is_geo_encoded, 1); $this->assertEqual($post->geo, '28.6889398,77.1618859'); $this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi'); $this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India'); // Distance between main post and its reply (Geocoding Process) $this->assertEqual($post->reply_retweet_distance, 1161); // Test 2: Checking Post for successful Geoencoding using "place" field // This post is retrieved from tu_encoded_locations $post = $post_dao->getPost('14914043658', 'twitter'); $this->assertEqual($post->is_geo_encoded, 1); $this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi'); $this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India'); // When reply is Not in DB, reply_retweet_distance is -1 $this->assertFalse($post_dao->isPostInDB('999999', 'twitter')); $this->assertEqual($post->reply_retweet_distance, -1); // Test 1: Checking Post for successful Geoencoding using "location" field (post had is_geo_encoded set to 3) $post = $post_dao->getPost('15338041815', 'twitter'); $this->assertEqual($post->geo, '19.017656,72.856178'); $this->assertEqual($post->place, NULL); $this->assertEqual($post->location, 'Mumbai, Maharashtra, India'); $this->assertEqual($post->is_geo_encoded, 1); // Test 2: Checking Post for successful Geoencoding using "location" field $post = $post_dao->getPost('15344199472', 'twitter'); $this->assertEqual($post->location, 'New Delhi, Delhi, India'); $this->assertEqual($post->is_geo_encoded, 1); // Distance between Post and Retweet (Geocoding Process) $this->assertEqual($post->reply_retweet_distance, 18); // When all three fields are filled, <geo> is given the most preference $post = $post_dao->getPost('11259110570', 'twitter'); $this->assertEqual($post->geo, '28.56213,77.165297'); $this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi'); $this->assertEqual($post->location, 'Vasant Vihar, Munirka, New Delhi, Delhi, India'); $this->assertEqual($post->is_geo_encoded, 1); // Distance between reply and post (Reverse Geocoding Process) $this->assertEqual($post->reply_retweet_distance, 14); // When only place and location are filled, <place> is given preference $post = $post_dao->getPost('15052338902', 'twitter'); $this->assertEqual($post->geo, '28.6889398,77.1618859'); $this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi'); $this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India'); $this->assertEqual($post->is_geo_encoded, 1); // Unsuccessful Geoencoding due to place field // NOTE: Not a test case encountered in real crawl $post = $post_dao->getPost('14913946516', 'twitter'); $this->assertEqual($post->geo, NULL); $this->assertEqual($post->place, 'abc'); $this->assertEqual($post->location, 'New Delhi'); $this->assertEqual($post->is_geo_encoded, 2); $this->assertEqual($post->reply_retweet_distance, 0); //Unsuccessful Geoencoding due to location field $post = $post_dao->getPost('15268690400', 'twitter'); $this->assertEqual($post->geo, NULL); $this->assertEqual($post->place, NULL); $this->assertEqual($post->location, 'abc'); $this->assertEqual($post->is_geo_encoded, 2); //Unsuccessful Geoencoding due to location field resulting in INVALID_REQUEST // Removing this test due to inconsistent behavior between testing environments for non-UTF8 chars and errors // like: // General error: 1366 Incorrect string value: '\xDC' for column 'location' at row 1 //$post = $post_dao->getPost('15244973830', 'twitter'); //$this->assertEqual($post->location, 'Ü'); //$this->assertEqual($post->is_geo_encoded, 5); //Unsuccessful Geoencoding due to all three fields being empty $post = $post_dao->getPost('15435434230', 'twitter'); $this->assertEqual($post->geo, NULL); $this->assertEqual($post->place, NULL); $this->assertEqual($post->location, NULL); $this->assertEqual($post->is_geo_encoded, 6); //Reverse Geoencoding when latitude and longitude are found in location field instead of geo field $post = $post_dao->getPost('13212618909', 'twitter'); $this->assertEqual($post->geo, '40.681839,-73.983734'); $this->assertEqual($post->place, NULL); $this->assertEqual($post->location, 'Boerum Hill, Brooklyn, NY, USA'); $this->assertEqual($post->is_geo_encoded, 1); // Retweet Distance in case of Reverse Geocoding Process $this->assertEqual($post->reply_retweet_distance, 11760); //Unsuccessful Geoencoding due to REQUEST_DENIED $post = $post_dao->getPost('12259110570', 'twitter'); $this->assertEqual($post->place, 'request_denied'); $this->assertEqual($post->is_geo_encoded, 4); //Unsuccessful Geoencoding due to OVER_QUERY_LIMIT $post = $post_dao->getPost('13259110570', 'twitter'); $this->assertEqual($post->place, 'over_query_limit'); $this->assertEqual($post->is_geo_encoded, 3); //After reaching OVER_QUERY_LIMIT, next posts are not geoencoded $post = $post_dao->getPost('15645301636', 'twitter'); $this->assertEqual($post->is_geo_encoded, 0); $post = $post_dao->getPost('11331235880', 'twitter'); $this->assertEqual($post->is_geo_encoded, 0); // Check up filling of tu_encoded_locations table $locations = $location_dao->getAllLocations(); $this->assertEqual(count($locations), 6); $this->assertEqual($locations[0]['short_name'], "28.602815 77.049136"); $this->assertEqual($locations[2]['short_name'], "Mumbai"); $this->assertEqual($locations[5]['short_name'], "40.681839 -73.983734"); }
public function testCrawlUpgrading() { // up app version $config = Config::getInstance(); $init_db_version = $config->getValue('THINKUP_VERSION'); $config->setValue('THINKUP_VERSION', $config->getValue('THINKUP_VERSION') + 10); //set a high version num $builders = $this->buildData(); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $crawler_plugin_registrar->registerPlugin('hellothinkup', 'HelloThinkUpPlugin'); $crawler_plugin_registrar->registerCrawlerPlugin('HelloThinkUpPlugin'); $this->simulateLogin('*****@*****.**', true); $this->expectException(new InstallerException('ThinkUp needs a database migration, so we are unable to run the crawler.')); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); // reset version $config->setValue('THINKUP_VERSION', $init_db_version); }