die("DMI-TCAT requires at least MySQL version 5.5.3 - please upgrade your server\n");
}
$archive_dbh = new PDO("mysql:host={$dbhost_from};dbname=" . $dbdatabase_from, $dbuser_from, $dbpass_from, array(PDO::MYSQL_ATTR_INIT_COMMAND => "set sql_mode='ALLOW_INVALID_DATES'"));
$archive_dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$dbh = new PDO("mysql:host={$dbhost_to};dbname=" . $dbdatabase_to, $dbuser_to, $dbpass_to, array(PDO::MYSQL_ATTR_INIT_COMMAND => "set sql_mode='ALLOW_INVALID_DATES'"));
$dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
if (empty($bin_name)) {
die("bin_name not set\n");
}
$querybin_id = queryManagerBinExists($bin_name);
// Old table, importing tweets from here
$query = $archive_dbh->prepare("SELECT * FROM " . $ytk_table);
$query->execute();
// create new tables
create_bin($bin_name, $dbh);
queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, 'import ytk', $queries);
// insert old data in new tables
while ($row = $query->fetch(PDO::FETCH_ASSOC)) {
$txt = $row['text'];
$hashtags = $urls = $mentions = $t = array();
$t["id"] = $row['id'];
$t["text"] = $txt;
$t["created_at"] = date("Y-m-d H:i:s", strtotime($row["created_at"]));
$t["from_user_name"] = $row['from_user'];
$t["from_user_id"] = $row['from_user_id'];
$t["from_user_lang"] = $row['iso_language_code'];
$t["from_user_tweetcount"] = null;
$t["from_user_followercount"] = null;
$t["from_user_friendcount"] = null;
$t["from_user_realname"] = null;
$t["source"] = $row['source'];
die("bin_name not set\n");
}
if (empty($keywords)) {
die("keywords not set\n");
}
if (dbserver_has_utf8mb4_support() == false) {
die("DMI-TCAT requires at least MySQL version 5.5.3 - please upgrade your server");
}
$querybin_id = queryManagerBinExists($bin_name, $cronjob);
$current_key = 0;
$tweetQueue = new TweetQueue();
// ----- connection -----
$dbh = pdo_connect();
create_bin($bin_name, $dbh);
$ratefree = 0;
queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, $type, explode("OR", $keywords));
search($keywords);
if ($tweetQueue->length() > 0) {
$tweetQueue->insertDB();
}
function search($keywords, $max_id = null)
{
global $twitter_keys, $current_key, $ratefree, $bin_name, $dbh, $tweetQueue;
$ratefree--;
if ($ratefree < 1 || $ratefree % 10 == 0) {
$keyinfo = getRESTKey($current_key, 'search', 'tweets');
$current_key = $keyinfo['key'];
$ratefree = $keyinfo['remaining'];
}
$tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret']));
$params = array('q' => $keywords, 'count' => 100);
include_once __DIR__ . '/../common/functions.php';
include_once __DIR__ . '/../capture/common/functions.php';
// specify the name of the bin here
$bin_name = '';
// specify dir with the user timelines (json)
$dir = '';
if (empty($bin_name)) {
die("bin_name not set\n");
}
if (dbserver_has_utf8mb4_support() == false) {
die("DMI-TCAT requires at least MySQL version 5.5.3 - please upgrade your server\n");
}
$querybin_id = queryManagerBinExists($bin_name);
$dbh = pdo_connect();
create_bin($bin_name, $dbh);
queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, 'import gnip');
$all_files = glob("{$dir}/*");
$all_users = $all_tweet_ids = array();
$tweets_processed = $tweets_failed = $tweets_success = 0;
$count = count($all_files);
$c = $count;
for ($i = 0; $i < $count; ++$i) {
$filepath = $all_files[$i];
process_json_file_timeline($filepath, $dbh);
print $c-- . "\n";
}
function process_json_file_timeline($filepath, $dbh)
{
print $filepath . "\n";
global $tweets_processed, $tweets_failed, $tweets_success, $all_tweet_ids, $all_users, $bin_name;
$tweetQueue = new TweetQueue();
$querybin_id = queryManagerBinExists($bin_name);
$dbh = pdo_connect();
create_bin($bin_name, $dbh);
$all_files = glob("{$dir}/*.json");
global $tweets_processed, $tweets_failed, $tweets_success, $valid_timeline, $empty_timeline, $invalid_timeline, $populated_timeline, $total_timeline, $all_users, $all_tweet_ids;
$all_users = $all_tweet_ids = array();
$tweets_processed = $tweets_failed = $tweets_success = $valid_timeline = $empty_timeline = $invalid_timeline = $populated_timeline = $total_timeline = 0;
$count = count($all_files);
$c = $count;
for ($i = 0; $i < $count; ++$i) {
$filepath = $all_files[$i];
print "processing {$filepath}\n";
process_json_file_timeline($filepath, $dbh);
print $c-- . "\n";
}
queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, $type, $queries);
function process_json_file_timeline($filepath, $dbh)
{
global $tweets_processed, $tweets_failed, $tweets_success, $valid_timeline, $empty_timeline, $invalid_timeline, $populated_timeline, $total_timeline, $all_tweet_ids, $all_users, $bin_name;
$tweetQueue = new TweetQueue();
$total_timeline++;
ini_set('auto_detect_line_endings', true);
$handle = @fopen($filepath, "r");
if ($handle) {
while (($buffer = fgets($handle, 40960)) !== false) {
$tweet = json_decode($buffer, true);
//var_export($tweet); print "\n\n";
$buffer = "";
$t = new Tweet();
$t->fromJSON($tweet);
if (!$t->isInBin($bin_name)) {
