private function _graph_multi($method = 'GET', $params = array(array()), $callback = "echo")
{
$mc = curl_multi_init();
// FIRST: Prepare the parameters for all the requests.
for ($i = 0; $i < count($params); $i++) {
$params[$i]['method'] = $method;
if (!isset($params[$i]['access_token'])) {
$params[$i]['access_token'] = $this->getAccessToken();
}
foreach (array_keys($params[$i]) as $j) {
if (!is_string($params[$i][$j])) {
$params[$i][$j] = json_encode($params[$i][$j]);
}
}
}
// SECOND: Grab the initial set of connections from the global queue
$i = 0;
try {
$connections = Facebook::getQueue()->shift(PriorityQueue::$POPCNT);
$this->log(date("G:i:s ") . "Grabbed elements; objects remaining in queue: " . Facebook::getQueue()->count() . " (highest level: " . Facebook::getQueue()->highestLevel() . ")");
} catch (Exception $e) {
//echo "_graph_mult() Exception when shifting: " . $e->getMessage() . "<br />";
ob_flush();
flush();
return;
}
// THIRD: Go through all of these connections, replace it if it refers to an existing file, then construct the cURL multi request
foreach ($connections as $i => $connection) {
$fname = Connection::createSafeName($this, $connection->getUrl());
while (file_exists($fname)) {
try {
$connectionarray = Facebook::getQueue()->shift(1);
$this->log("Replacing connection " . $i . " pointing to " . $fname);
$connections[$i] = $connection = $connectionarray[0];
} catch (Exception $e) {
unset($connection);
fprintf($this->getLogFd(), "_graph_mult() Exception when shifting to grab replacement: " . $e->getMessage() . "\n");
// If the queue is empty, bail out
if (Facebook::getQueue()->count() < 1) {
break 2;
}
continue;
}
fprintf($this->getLogFd(), "Replaced by connection to " . $connection->getUrl() . ", " . Facebook::getQueue()->count() . " elements left in queue\n");
$fname = Connection::createSafeName($this, $connection->getUrl());
}
if (FALSE === strpos($connection->getUrl(), "http")) {
$url = $this->getUrl('graph', $connection->getUrl());
} else {
$url = $connection->getUrl();
}
$this->log("Constructing request to " . $url);
$ch = $this->constructRequest($url, $params[$i]);
if (!isset($params[$i]['access_token'])) {
$this->log("[ERROR] No access token set on request to " . $url . ", have to discard it.");
} else {
$channels[] = $ch;
// What is this i don't even
curl_multi_add_handle($mc, $ch);
}
$i++;
}
// FOURTH: Execute the request(s) and check for errors.
do {
$execret = curl_multi_exec($mc, $running);
} while ($execret == CURLM_CALL_MULTI_PERFORM);
while ($running && $execret == CURLM_OK) {
$ready = curl_multi_select($mc);
if ($ready != -1) {
do {
$execret = curl_multi_exec($mc, $running);
} while ($execret == CURLM_CALL_MULTI_PERFORM);
}
}
if ($execret != CURLM_OK) {
trigger_error("makeRequest_multi() Curl multi read error {$execret}\n", E_USER_WARNING);
}
// FIFTH: Go through returned connections, pass the result to the callback and clean up cURL_multi
$index = 0;
foreach ($connections as $i => $connection) {
$this->log("Handling returned connection for " . $connection->getUrl());
//echo "makeRequest_multi() Trying to read data from a connection<br />"
if ($channels[$index] == NULL) {
$curlerror = "Channel is NULL, there probably was no access token set. Continuing regardless.<br />";
} else {
$curlerror = curl_error($channels[$index]);
}
if ("" == $curlerror) {
$handlerinfo = curl_getinfo($channels[$index], CURLINFO_EFFECTIVE_URL);
$this->log("URL on the cURL handler was " . $handlerinfo);
$content = curl_multi_getcontent($channels[$index]);
call_user_func($callback, $connection, curl_multi_getcontent($channels[$index]), $this);
//echo "Back from callback.<br />";
} else {
print "makeRequest_multi() Curl error on handle {$i}: {$curlerror}\n";
}
if ($channels[$index] != NULL) {
curl_multi_remove_handle($mc, $channels[$index]);
//curl_close($channels[$index]);
}
$index++;
}
curl_multi_close($mc);
//echo "---------------------------------------------GRAPH_MULTI END---------------------------------------------<br />";
}
/**
* Handles downloaded data.
* The function name is a bit of a misnomer, as it does not actually fetch the contents when using api_multi. This has been done ahead of time (unless we're using the old api() calls).
* This function writes the downloaded content (json-ified, if it's not a Picture) into its respective output file, causes the new object to create its connections and add them to the queue.
* @param $facebook A Facebook instance to use for fetching data.
*/
public function fetch($facebook)
{
// If the folder for our output files doesn't exist yet, create it
if (!is_dir("../tmp/" . $facebook->getUnique())) {
mkdir("../tmp/" . $facebook->getUnique());
}
// Create a safe file name. Simply replaces all slashes, actually.
//TODO: Write this platform-independent-safe
$fname = Connection::createSafeName($facebook, $this->url);
//$fname = $facebook->getUnique() . "/" .strtr($this->url, "/", "~") . ".request";
// Is this a Picture? If so, we don't process the content but simply write it into a file that has base64($url) as its filename.
if ($this->type == 'Picture') {
fprintf($facebook->getLogFd(), "Writing picture with filesize " . strlen($this->json) . "\n");
if (!file_exists("../tmp/" . $facebook->getUnique() . "/" . base64_encode($this->url))) {
file_put_contents("../tmp/" . $facebook->getUnique() . "/" . base64_encode($this->url), $this->json);
}
return new Picture("", 0);
}
try {
// Check if the file already exists; if so, throw an exception
if (file_exists($fname)) {
throw new Exception("File " . $fname . " already exists.");
}
// If json is empty, we haven't fetched any content yet, which means that we're using the old API.
// So let's just use the old api() call. This one also does an implicit json_decode(), so we don't have to perform that anymore.
if (strlen($this->json) < 1) {
$this->json = $facebook->api($this->url);
} else {
$facebook->log("[RESPONSE] Response's json is larger than 0");
$this->json = json_decode($this->json, true);
}
// Check if the Graph API returned an error.
if (isset($this->json['error'])) {
//echo "fetch() FB Error:<br />";
//print_r($this->json);
throw new Exception("fb error: " . json_encode($this->json['error']));
}
} catch (Exception $e) {
//echo "fetch() Exception occurred (" . $e->getMessage() . "), continuing anyway, handling as empty Picture";
//ob_flush();
//flush()
$facebook->log("[ERROR] fetch() Exception occurred (" . $e->getMessage() . "), continuing anyway, handling as empty Picture");
// This "empty picture" is nearly an empty object. It has no connections and should therefore be completely neutral to the rest of the process.
return new Picture("", 0);
}
// Open the output file for writing
$facebook->log("[FILE] Creating file " . $fname);
$fp = fopen($fname, "w");
// Write the json data - in text form - to the file
//fwrite($fp, print_r($this->json, TRUE));
fwrite($fp, json_encode($this->json));
// Close the output file again.
fclose($fp);
if (isset($this->json['paging']) && isset($this->json['paging']['next'])) {
$queue = new PriorityQueue();
$url = substr($this->json['paging']['next'], strpos($this->json['paging']['next'], "com/") + 4);
$facebook->log("[DEBUG] Adding paging URL " . $url);
$queue->unshift(new Connection($url, $this->depth, $this->type, true), 100);
Facebook::getQueue()->merge($queue);
} else {
$facebook->log("[DEBUG] No paging or next");
}
// If the data is not "right there" at the topmost level but nested in the data part, replace our internal variable with the actual payload.
if (isset($this->json['data'])) {
$this->json = $this->json['data'];
}
// Check if there are multiple objects stored in the received json
if ($this->multiplicity) {
$retval = array();
// Handle each object in json
foreach ($this->json as $item) {
// First, the two "meta-types" Profile and Taggable; they're not actual types, but they can determine which of their subtypes is the appropriate one with their static getInstance() method.
if ($this->type == 'Profile') {
array_push($retval, Profile::getInstance($item, $this->depth));
} else {
if ($this->type == 'Taggable') {
array_push($retval, Taggable::getInstance($item, $this->depth));
} else {
// Slight PHP magic: $this->type is a string that contains a class name, i.e. we construct an object whose class name is specified by that field.
array_push($retval, new $this->type($item, $this->depth));
}
}
if ($this->type == 'User') {
$facebook->log('Created a user.');
}
//$facebook->log(print_r($item));
}
$fullnull = true;
//Performing getConnections() now, adding everything into the big static queue
// Also, we check if all getConnections() return NULL
foreach ($retval as $item) {
if (NULL != $item->getConnections()) {
$fullnull = false;
}
}
// All getConnections() have returned NULL, which means that the depth is too high (or deep?). So it's time for us to return NULL, too, in order to let the recursion end at this point (actually, it doesn't end, it just switches to a different part of the recursion tree).
if ($fullnull) {
return NULL;
}
// Return the array with all parsed objects
return $retval;
} else {
// Same as before: Call getInstance for the meta-types, otherwise create an object with the type $this->type.
if ($this->type == 'Profile') {
$retval = Profile::getInstance($this->json, $this->depth);
} else {
if ($this->type == 'Taggable') {
$retval = Taggable::getInstance($this->json, $this->depth);
} else {
$retval = new $this->type($this->json, $this->depth);
}
}
//Performing getConnections() now, adding this element's connection into the big static queue
if (NULL == $retval->getConnections()) {
return NULL;
}
// Return the parsed object
return $retval;
}
}
function readNode($facebook, $parent, $sendid)
{
//echo "readNode()<br />";
$connections = $parent->getConnections();
echo "<pre>";
while (Facebook::getQueue()->count() > 0) {
$facebook->api_multi('GET', Connection::createEmptyArray(), array("Connection", "recursor"));
$facebook->log(date("G:i:s D M j T Y") . " Returned into readNode(), " . Facebook::getQueue()->count() . " elements left, let's get back in there! Highest Level: " . Facebook::getQueue()->highestLevel());
echo date("G:i:s D M j T Y") . " " . Facebook::getQueue()->count() . " elements left. Highest Level: " . Facebook::getQueue()->highestLevel() . "<br/>";
if (Facebook::getQueue()->highestLevel() < 3 || Facebook::getQueue()->count() < 3) {
$facebook->log("Finished. highestLevel: " . Facebook::getQueue()->highestLevel());
$facebook->log("Finished " . date("G:i:s D M j T Y"));
echo "</pre><h4>Finished " . date("G:i:s D M j T Y") . "</h4>";
$remaining = print_r(Facebook::getQueue(), true);
$facebook->log($remaining);
break;
}
flushOutput();
}
// Compress the gathered socialsnapshot
// Tar and compress the logfile and folder
// Check if the token is valid (must not contain anything but alphanumeric plus _) and if the folder and logs for this run really exist
if (0 != preg_match("/[^\\w]/", $sendid) || !file_exists("../tmp/folder" . $sendid) || !file_exists("../tmp/log" . $sendid)) {
// Die otherwise
die("Compression Failed: Could not find according socialsnapshot and log.");
} else {
exec("cd ../tmp && tar -hcjf ../tarballs/" . $sendid . ".tar.bz2 log" . $sendid . " folder" . $sendid . " > /dev/null");
exec("touch ../tmp/" . $sendid . ".finished > /dev/null");
exec("rm -r ../tmp/logsnapshot" . $sendid . " ../tmp/folder" . $sendid . " > /dev/null");
exec("rm -r ../tmp/" . $facebook->getUnique() . " > /dev/null");
}
//If optional analyse script is available, run it
$analysescript = "/opt/FBSnapshotLoader/scripts/analysesnapshot.sh";
if (file_exists($analysescript)) {
$installpath = realpath('../');
$snapshotfile = realpath('../tarballs/' . $sendid . '.tar.bz2');
$downloadurl = 'https://' . $_SERVER["HTTP_HOST"] . '/SocialSnapshot/downloads';
$analysecommand = 'LANG=en_US.utf-8; ' . $analysescript . ' ' . $snapshotfile . ' ' . $downloadurl . ' ' . $installpath . ' > /dev/null 2>&1 &';
//echo $analysecommand;
exec($analysecommand);
}
}
