die("DMI-TCAT requires at least MySQL version 5.5.3 - please upgrade your server\n"); } $archive_dbh = new PDO("mysql:host={$dbhost_from};dbname=" . $dbdatabase_from, $dbuser_from, $dbpass_from, array(PDO::MYSQL_ATTR_INIT_COMMAND => "set sql_mode='ALLOW_INVALID_DATES'")); $archive_dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $dbh = new PDO("mysql:host={$dbhost_to};dbname=" . $dbdatabase_to, $dbuser_to, $dbpass_to, array(PDO::MYSQL_ATTR_INIT_COMMAND => "set sql_mode='ALLOW_INVALID_DATES'")); $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); if (empty($bin_name)) { die("bin_name not set\n"); } $querybin_id = queryManagerBinExists($bin_name); // Old table, importing tweets from here $query = $archive_dbh->prepare("SELECT * FROM " . $ytk_table); $query->execute(); // create new tables create_bin($bin_name, $dbh); queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, 'import ytk', $queries); // insert old data in new tables while ($row = $query->fetch(PDO::FETCH_ASSOC)) { $txt = $row['text']; $hashtags = $urls = $mentions = $t = array(); $t["id"] = $row['id']; $t["text"] = $txt; $t["created_at"] = date("Y-m-d H:i:s", strtotime($row["created_at"])); $t["from_user_name"] = $row['from_user']; $t["from_user_id"] = $row['from_user_id']; $t["from_user_lang"] = $row['iso_language_code']; $t["from_user_tweetcount"] = null; $t["from_user_followercount"] = null; $t["from_user_friendcount"] = null; $t["from_user_realname"] = null; $t["source"] = $row['source'];
die("bin_name not set\n"); } if (empty($keywords)) { die("keywords not set\n"); } if (dbserver_has_utf8mb4_support() == false) { die("DMI-TCAT requires at least MySQL version 5.5.3 - please upgrade your server"); } $querybin_id = queryManagerBinExists($bin_name, $cronjob); $current_key = 0; $tweetQueue = new TweetQueue(); // ----- connection ----- $dbh = pdo_connect(); create_bin($bin_name, $dbh); $ratefree = 0; queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, $type, explode("OR", $keywords)); search($keywords); if ($tweetQueue->length() > 0) { $tweetQueue->insertDB(); } function search($keywords, $max_id = null) { global $twitter_keys, $current_key, $ratefree, $bin_name, $dbh, $tweetQueue; $ratefree--; if ($ratefree < 1 || $ratefree % 10 == 0) { $keyinfo = getRESTKey($current_key, 'search', 'tweets'); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; } $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); $params = array('q' => $keywords, 'count' => 100);
include_once __DIR__ . '/../common/functions.php'; include_once __DIR__ . '/../capture/common/functions.php'; // specify the name of the bin here $bin_name = ''; // specify dir with the user timelines (json) $dir = ''; if (empty($bin_name)) { die("bin_name not set\n"); } if (dbserver_has_utf8mb4_support() == false) { die("DMI-TCAT requires at least MySQL version 5.5.3 - please upgrade your server\n"); } $querybin_id = queryManagerBinExists($bin_name); $dbh = pdo_connect(); create_bin($bin_name, $dbh); queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, 'import gnip'); $all_files = glob("{$dir}/*"); $all_users = $all_tweet_ids = array(); $tweets_processed = $tweets_failed = $tweets_success = 0; $count = count($all_files); $c = $count; for ($i = 0; $i < $count; ++$i) { $filepath = $all_files[$i]; process_json_file_timeline($filepath, $dbh); print $c-- . "\n"; } function process_json_file_timeline($filepath, $dbh) { print $filepath . "\n"; global $tweets_processed, $tweets_failed, $tweets_success, $all_tweet_ids, $all_users, $bin_name; $tweetQueue = new TweetQueue();
$querybin_id = queryManagerBinExists($bin_name); $dbh = pdo_connect(); create_bin($bin_name, $dbh); $all_files = glob("{$dir}/*.json"); global $tweets_processed, $tweets_failed, $tweets_success, $valid_timeline, $empty_timeline, $invalid_timeline, $populated_timeline, $total_timeline, $all_users, $all_tweet_ids; $all_users = $all_tweet_ids = array(); $tweets_processed = $tweets_failed = $tweets_success = $valid_timeline = $empty_timeline = $invalid_timeline = $populated_timeline = $total_timeline = 0; $count = count($all_files); $c = $count; for ($i = 0; $i < $count; ++$i) { $filepath = $all_files[$i]; print "processing {$filepath}\n"; process_json_file_timeline($filepath, $dbh); print $c-- . "\n"; } queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, $type, $queries); function process_json_file_timeline($filepath, $dbh) { global $tweets_processed, $tweets_failed, $tweets_success, $valid_timeline, $empty_timeline, $invalid_timeline, $populated_timeline, $total_timeline, $all_tweet_ids, $all_users, $bin_name; $tweetQueue = new TweetQueue(); $total_timeline++; ini_set('auto_detect_line_endings', true); $handle = @fopen($filepath, "r"); if ($handle) { while (($buffer = fgets($handle, 40960)) !== false) { $tweet = json_decode($buffer, true); //var_export($tweet); print "\n\n"; $buffer = ""; $t = new Tweet(); $t->fromJSON($tweet); if (!$t->isInBin($bin_name)) {