/** * Destroy Config, Webapp, $_SESSION, $_POST, $_GET, $_REQUEST */ public function tearDown() { Config::destroyInstance(); PluginRegistrarWebapp::destroyInstance(); PluginRegistrarCrawler::destroyInstance(); if (isset($_SESSION)) { $this->unsetArray($_SESSION); } $this->unsetArray($_POST); $this->unsetArray($_GET); $this->unsetArray($_REQUEST); $this->unsetArray($_SERVER); $this->unsetArray($_FILES); Loader::unregister(); $backup_dir = FileDataManager::getBackupPath(); if (file_exists($backup_dir)) { try { @exec('cd ' . $backup_dir . '; rm -rf *'); rmdir($backup_dir); // won't delete if has files } catch (Exception $e) { } } $data_dir = FileDataManager::getDataPath(); if (file_exists($data_dir . 'compiled_view')) { try { @exec('cd ' . $data_dir . '; rm -rf compiled_view'); } catch (Exception $e) { } } parent::tearDown(); }
public function setUp() { parent::setUp(); $this->webapp = PluginRegistrarWebapp::getInstance(); $this->crawler = PluginRegistrarCrawler::getInstance(); $this->webapp->registerPlugin('twitter', 'TwitterPlugin'); $this->crawler->registerCrawlerPlugin('TwitterPlugin'); $this->webapp->setActivePlugin('twitter'); $this->logger = Logger::getInstance(); }
public function authControl() { if ($this->isAPICall()) { // If the request comes from an API call, output JSON instead of HTML $this->setContentType('application/json; charset=UTF-8'); } else { $this->setContentType('text/html; charset=UTF-8'); $this->setViewTemplate('crawler.run-top.tpl'); echo $this->generateView(); $config = Config::getInstance(); $config->setValue('log_location', false); //this forces output to just echo to page $logger = Logger::getInstance(); $logger->close(); } try { $logger = Logger::getInstance(); if (isset($_GET['log']) && $_GET['log'] == 'full') { $logger->setVerbosity(Logger::ALL_MSGS); echo '<pre style="font-family:Courier;font-size:10px;">'; } else { $logger->setVerbosity(Logger::USER_MSGS); $logger->enableHTMLOutput(); } $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); //close session so that it's not locked by long crawl session_write_close(); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); $logger->close(); } catch (CrawlerLockedException $e) { if ($this->isAPICall()) { // Will be caught and handled in ThinkUpController::go() throw $e; } else { // Will appear in the textarea of the HTML page echo '<td></td><td>' . $e->getMessage() . '</td><td></td>'; } } if ($this->isAPICall()) { echo json_encode((object) array('result' => 'success')); } else { $this->setViewTemplate('crawler.run-bottom.tpl'); echo $this->generateView(); } }
public function control() { $output = ""; $authorized = false; if (isset($this->argc) && $this->argc > 1) { // check for CLI credentials $this->content_type = 'text/plain'; $session = new Session(); $username = $this->argv[1]; if ($this->argc > 2) { $pw = $this->argv[2]; } else { $pw = getenv('THINKUP_PASSWORD'); } $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail($username); if ($owner_dao->isOwnerAuthorized($username, $pw)) { $authorized = true; Session::completeLogin($owner); } else { $output = "ERROR: Incorrect username and password."; } } else { // check user is logged in on the web if ($this->isLoggedIn()) { $authorized = true; } else { $output = "ERROR: Invalid or missing username and password."; } } if ($authorized) { $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); } return $output; }
/** * Provided only for tests that want to kill object in tearDown() */ public static function destroyInstance() { if (isset(self::$instance)) { self::$instance = null; } }
*/ /** * * ThinkUp/webapp/plugins/facebook/controller/facebook.php * * Copyright (c) 2009-2015 Gina Trapani * * LICENSE: * * This file is part of ThinkUp (http://thinkup.com). * * ThinkUp is free software: you can redistribute it and/or modify it under the terms of the GNU General Public * License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any * later version. * * ThinkUp is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. * * You should have received a copy of the GNU General Public License along with ThinkUp. If not, see * <http://www.gnu.org/licenses/>. * * @author Gina Trapani <ginatrapani[at]gmail[dot]com> * @license http://www.gnu.org/licenses/gpl.html * @copyright 2009-2015 Gina Trapani */ $webapp_plugin_registrar = PluginRegistrarWebapp::getInstance(); $webapp_plugin_registrar->registerPlugin('facebook', 'FacebookPlugin'); $webapp_plugin_registrar->registerPlugin('facebook page', 'FacebookPlugin'); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $crawler_plugin_registrar->registerCrawlerPlugin('FacebookPlugin');
public function testBitlyCrawl() { $builders = $this->buildBitlyData(); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $config = Config::getInstance(); //use fake Bitly API key $builders[] = FixtureBuilder::build('options', array('namespace' => OptionDAO::PLUGIN_OPTIONS . '-' . $this->plugin_id, 'option_name' => 'bitly_api_key', 'option_value' => 'dummykey')); //use fake Bitly login name $builder[] = FixtureBuilder::build('options', array('namespace' => OptionDAO::PLUGIN_OPTIONS . '-' . $this->plugin_id, 'option_name' => 'bitly_login', 'option_value' => 'bitly123')); $this->simulateLogin('*****@*****.**', true); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); $link_dao = DAOFactory::getDAO('LinkDAO'); $link = $link_dao->getLinkById(43); $this->assertEqual($link->expanded_url, 'http://static.ak.fbcdn.net/rsrc.php/zw/r/ZEKh4ZZQY74.png'); $this->assertEqual($link->title, 'Bitly Test URL'); $this->assertEqual($link->error, ''); $link = $link_dao->getLinkById(42); $this->assertEqual($link->expanded_url, 'http://bitly.com/a/warning?url=http%3a%2f%2fwww%2ealideas%2ecom%2f&hash=41'); $this->assertEqual($link->error, 'No response from http://bit.ly API'); $link = $link_dao->getLinkById(41); $this->debug($link->url); $this->assertEqual($link->expanded_url, 'http://static.ak.fbcdn.net/rsrc.php/zw/r/ZEKh4ZZQY74.png'); $this->assertEqual($link->error, 'No response from http://bit.ly API'); }
public function testOrderOfPluginsWithoutInsightGenerator() { $plugin_registrar = new PluginRegistrarCrawler(); $plugin_registrar->registerCrawlerPlugin('TwitterPlugin', true); $plugin_registrar->registerCrawlerPlugin('FacebookPlugin', true); $plugin_registrar->registerCrawlerPlugin('FoursquarePlugin', false); $plugin_registrar->registerCrawlerPlugin('YouTubePlugin', false); // Order the plugins and check they're in the right order $plugin_registrar->orderPlugins('crawl'); $plugins = $plugin_registrar->getObjectFunctionCallbacks(); $this->assertNotNull($plugins); $crawl_plugins = $plugins['crawl']; $this->assertEqual($crawl_plugins[0][0], 'TwitterPlugin'); $this->assertEqual($crawl_plugins[1][0], 'FacebookPlugin'); $this->assertEqual($crawl_plugins[2][0], 'FoursquarePlugin'); $this->assertEqual($crawl_plugins[3][0], 'YouTubePlugin'); }
public function testGeoEncoderCrawl() { $this->simulateLogin('*****@*****.**', true); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); //the crawler closes the log so we have to re-open it $logger = Logger::getInstance(); $post_dao = DAOFactory::getDAO('PostDAO'); $location_dao = DAOFactory::getDAO('LocationDAO'); // Test 1: Checking Post for Successful Reverse Geoencoding $this->assertTrue($post_dao->isPostInDB('15645300636', 'twitter')); $post = $post_dao->getPost('15645300636', 'twitter'); $this->assertEqual($post->is_geo_encoded, 1); $this->assertEqual($post->geo, '28.602815,77.049136'); $this->assertEqual($post->location, 'Sector 4, New Delhi, Haryana, India'); // Since this is just a post, reply_retweet_distance is 0 $this->assertEqual($post->reply_retweet_distance, 0); // Test 2: Checking Post for successful Reverse Geoencoding $post = $post_dao->getPost('15219161227', 'twitter'); $this->assertEqual($post->is_geo_encoded, 1); $this->assertEqual($post->geo, '28.56213,77.165297'); $this->assertEqual($post->location, 'Vasant Vihar, Munirka, New Delhi, Delhi, India'); // Test: Example of unsuccessful geoencoding resulting out of INVALID_REQUEST. // NOTE: Not a test case encountered in actual crawl $post = $post_dao->getPost('15331235880', 'twitter'); $this->assertEqual($post->is_geo_encoded, 5); $this->assertEqual($post->geo, '28.60abc2815 77.049136'); // Test 1: Checking Post for successful Geoencoding using "place" field $post = $post_dao->getPost('15052338902', 'twitter'); $this->assertEqual($post->is_geo_encoded, 1); $this->assertEqual($post->geo, '28.6889398,77.1618859'); $this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi'); $this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India'); // Distance between main post and its reply (Geocoding Process) $this->assertEqual($post->reply_retweet_distance, 1161); // Test 2: Checking Post for successful Geoencoding using "place" field // This post is retrieved from tu_encoded_locations $post = $post_dao->getPost('14914043658', 'twitter'); $this->assertEqual($post->is_geo_encoded, 1); $this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi'); $this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India'); // When reply is Not in DB, reply_retweet_distance is -1 $this->assertFalse($post_dao->isPostInDB('999999', 'twitter')); $this->assertEqual($post->reply_retweet_distance, -1); // Test 1: Checking Post for successful Geoencoding using "location" field (post had is_geo_encoded set to 3) $post = $post_dao->getPost('15338041815', 'twitter'); $this->assertEqual($post->geo, '19.017656,72.856178'); $this->assertEqual($post->place, NULL); $this->assertEqual($post->location, 'Mumbai, Maharashtra, India'); $this->assertEqual($post->is_geo_encoded, 1); // Test 2: Checking Post for successful Geoencoding using "location" field $post = $post_dao->getPost('15344199472', 'twitter'); $this->assertEqual($post->location, 'New Delhi, Delhi, India'); $this->assertEqual($post->is_geo_encoded, 1); // Distance between Post and Retweet (Geocoding Process) $this->assertEqual($post->reply_retweet_distance, 18); // When all three fields are filled, <geo> is given the most preference $post = $post_dao->getPost('11259110570', 'twitter'); $this->assertEqual($post->geo, '28.56213,77.165297'); $this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi'); $this->assertEqual($post->location, 'Vasant Vihar, Munirka, New Delhi, Delhi, India'); $this->assertEqual($post->is_geo_encoded, 1); // Distance between reply and post (Reverse Geocoding Process) $this->assertEqual($post->reply_retweet_distance, 14); // When only place and location are filled, <place> is given preference $post = $post_dao->getPost('15052338902', 'twitter'); $this->assertEqual($post->geo, '28.6889398,77.1618859'); $this->assertEqual($post->place, 'Sector 8, R.K. Puram, New Delhi'); $this->assertEqual($post->location, 'Keshav Puram Metro Station, Maharaja Nahar Singh Marg, New Delhi, Delhi, India'); $this->assertEqual($post->is_geo_encoded, 1); // Unsuccessful Geoencoding due to place field // NOTE: Not a test case encountered in real crawl $post = $post_dao->getPost('14913946516', 'twitter'); $this->assertEqual($post->geo, NULL); $this->assertEqual($post->place, 'abc'); $this->assertEqual($post->location, 'New Delhi'); $this->assertEqual($post->is_geo_encoded, 2); $this->assertEqual($post->reply_retweet_distance, 0); //Unsuccessful Geoencoding due to location field $post = $post_dao->getPost('15268690400', 'twitter'); $this->assertEqual($post->geo, NULL); $this->assertEqual($post->place, NULL); $this->assertEqual($post->location, 'abc'); $this->assertEqual($post->is_geo_encoded, 2); //Unsuccessful Geoencoding due to location field resulting in INVALID_REQUEST // Removing this test due to inconsistent behavior between testing environments for non-UTF8 chars and errors // like: // General error: 1366 Incorrect string value: '\xDC' for column 'location' at row 1 //$post = $post_dao->getPost('15244973830', 'twitter'); //$this->assertEqual($post->location, 'Ü'); //$this->assertEqual($post->is_geo_encoded, 5); //Unsuccessful Geoencoding due to all three fields being empty $post = $post_dao->getPost('15435434230', 'twitter'); $this->assertEqual($post->geo, NULL); $this->assertEqual($post->place, NULL); $this->assertEqual($post->location, NULL); $this->assertEqual($post->is_geo_encoded, 6); //Reverse Geoencoding when latitude and longitude are found in location field instead of geo field $post = $post_dao->getPost('13212618909', 'twitter'); $this->assertEqual($post->geo, '40.681839,-73.983734'); $this->assertEqual($post->place, NULL); $this->assertEqual($post->location, 'Boerum Hill, Brooklyn, NY, USA'); $this->assertEqual($post->is_geo_encoded, 1); // Retweet Distance in case of Reverse Geocoding Process $this->assertEqual($post->reply_retweet_distance, 11760); //Unsuccessful Geoencoding due to REQUEST_DENIED $post = $post_dao->getPost('12259110570', 'twitter'); $this->assertEqual($post->place, 'request_denied'); $this->assertEqual($post->is_geo_encoded, 4); //Unsuccessful Geoencoding due to OVER_QUERY_LIMIT $post = $post_dao->getPost('13259110570', 'twitter'); $this->assertEqual($post->place, 'over_query_limit'); $this->assertEqual($post->is_geo_encoded, 3); //After reaching OVER_QUERY_LIMIT, next posts are not geoencoded $post = $post_dao->getPost('15645301636', 'twitter'); $this->assertEqual($post->is_geo_encoded, 0); $post = $post_dao->getPost('11331235880', 'twitter'); $this->assertEqual($post->is_geo_encoded, 0); // Check up filling of tu_encoded_locations table $locations = $location_dao->getAllLocations(); $this->assertEqual(count($locations), 6); $this->assertEqual($locations[0]['short_name'], "28.602815 77.049136"); $this->assertEqual($locations[2]['short_name'], "Mumbai"); $this->assertEqual($locations[5]['short_name'], "40.681839 -73.983734"); }
public function testCrawlUpgrading() { // up app version $config = Config::getInstance(); $init_db_version = $config->getValue('THINKUP_VERSION'); $config->setValue('THINKUP_VERSION', $config->getValue('THINKUP_VERSION') + 10); //set a high version num $builders = $this->buildData(); $crawler_plugin_registrar = PluginRegistrarCrawler::getInstance(); $crawler_plugin_registrar->registerPlugin('hellothinkup', 'HelloThinkUpPlugin'); $crawler_plugin_registrar->registerCrawlerPlugin('HelloThinkUpPlugin'); $this->simulateLogin('*****@*****.**', true); $this->expectException(new InstallerException('ThinkUp needs a database migration, so we are unable to run the crawler.')); $crawler_plugin_registrar->runRegisteredPluginsCrawl(); // reset version $config->setValue('THINKUP_VERSION', $init_db_version); }