static function purge($urlArr) { global $wgSquidServers, $wgHTCPMulticastAddress, $wgHTCPPort; /*if ( (@$wgSquidServers[0]) == 'echo' ) { echo implode("<br />\n", $urlArr) . "<br />\n"; return; }*/ if (!$urlArr) { return; } if ($wgHTCPMulticastAddress && $wgHTCPPort) { return SquidUpdate::HTCPPurge($urlArr); } wfProfileIn(__METHOD__); $maxSocketsPerSquid = 8; // socket cap per Squid $urlsPerSocket = 400; // 400 seems to be a good tradeoff, opening a socket takes a while $socketsPerSquid = ceil(count($urlArr) / $urlsPerSocket); if ($socketsPerSquid > $maxSocketsPerSquid) { $socketsPerSquid = $maxSocketsPerSquid; } $pool = new SquidPurgeClientPool(); $chunks = array_chunk($urlArr, ceil(count($urlArr) / $socketsPerSquid)); foreach ($wgSquidServers as $server) { foreach ($chunks as $chunk) { $client = new SquidPurgeClient($server); foreach ($chunk as $url) { $client->queuePurge($url); } $pool->addClient($client); } } $pool->run(); wfProfileOut(__METHOD__); }
/** * Purges a list of Squids defined in $wgSquidServers. * $urlArr should contain the full URLs to purge as values * (example: $urlArr[] = 'http://my.host/something') * XXX report broken Squids per mail or log * * @param $urlArr array * @return void */ static function purge($urlArr) { global $wgSquidServers, $wgHTCPMulticastRouting; if (!$urlArr) { return; } wfDebug("Squid purge: " . implode(' ', $urlArr) . "\n"); if ($wgHTCPMulticastRouting) { SquidUpdate::HTCPPurge($urlArr); } wfProfileIn(__METHOD__); $urlArr = array_unique($urlArr); // Remove duplicates $maxSocketsPerSquid = 8; // socket cap per Squid $urlsPerSocket = 400; // 400 seems to be a good tradeoff, opening a socket takes a while $socketsPerSquid = ceil(count($urlArr) / $urlsPerSocket); if ($socketsPerSquid > $maxSocketsPerSquid) { $socketsPerSquid = $maxSocketsPerSquid; } $pool = new SquidPurgeClientPool(); $chunks = array_chunk($urlArr, ceil(count($urlArr) / $socketsPerSquid)); foreach ($wgSquidServers as $server) { foreach ($chunks as $chunk) { $client = new SquidPurgeClient($server); foreach ($chunk as $url) { $client->queuePurge($url); } $pool->addClient($client); } } $pool->run(); wfProfileOut(__METHOD__); }
static function purge($urlArr) { global $wgSquidServers, $wgHTCPMulticastAddress, $wgHTCPPort; /*if ( (@$wgSquidServers[0]) == 'echo' ) { echo implode("<br />\n", $urlArr) . "<br />\n"; return; }*/ if (empty($urlArr)) { return; } if ($wgHTCPMulticastAddress && $wgHTCPPort) { return SquidUpdate::HTCPPurge($urlArr); } $fname = 'SquidUpdate::purge'; wfProfileIn($fname); $maxsocketspersquid = 8; // socket cap per Squid $urlspersocket = 400; // 400 seems to be a good tradeoff, opening a socket takes a while $firsturl = SquidUpdate::expand($urlArr[0]); unset($urlArr[0]); $urlArr = array_values($urlArr); $sockspersq = max(ceil(count($urlArr) / $urlspersocket), 1); if ($sockspersq == 1) { /* the most common case */ $urlspersocket = count($urlArr); } else { if ($sockspersq > $maxsocketspersquid) { $urlspersocket = ceil(count($urlArr) / $maxsocketspersquid); $sockspersq = $maxsocketspersquid; } } $totalsockets = count($wgSquidServers) * $sockspersq; $sockets = array(); /* this sets up the sockets and tests the first socket for each server. */ for ($ss = 0; $ss < count($wgSquidServers); $ss++) { $failed = false; $so = 0; while ($so < $sockspersq && !$failed) { if ($so == 0) { /* first socket for this server, do the tests */ @(list($server, $port) = explode(':', $wgSquidServers[$ss])); if (!isset($port)) { $port = 80; } #$this->debug("Opening socket to $server:$port"); $error = $errstr = false; $socket = @fsockopen($server, $port, $error, $errstr, 3); #$this->debug("\n"); if (!$socket) { $failed = true; $totalsockets -= $sockspersq; } else { $msg = 'PURGE ' . $firsturl . " HTTP/1.0\r\n" . "Connection: Keep-Alive\r\n\r\n"; #$this->debug($msg); @fputs($socket, $msg); #$this->debug("..."); $res = @fread($socket, 512); #$this->debug("\n"); /* Squid only returns http headers with 200 or 404 status, if there's more returned something's wrong */ if (strlen($res) > 250) { fclose($socket); $failed = true; $totalsockets -= $sockspersq; } else { @stream_set_blocking($socket, false); $sockets[] = $socket; } } } else { /* open the remaining sockets for this server */ list($server, $port) = explode(':', $wgSquidServers[$ss]); if (!isset($port)) { $port = 80; } $sockets[$so + 1] = @fsockopen($server, $port, $error, $errstr, 2); @stream_set_blocking($sockets[$so + 1], false); } $so++; } } if ($urlspersocket > 0) { /* now do the heavy lifting. The fread() relies on Squid returning only the headers */ for ($r = 0; $r < $urlspersocket; $r++) { for ($s = 0; $s < $totalsockets; $s++) { if ($r != 0) { $res = ''; $esc = 0; while (strlen($res) < 100 && $esc < 200) { $res .= @fread($sockets[$s], 512); $esc++; usleep(20); } } $urindex = $r + $urlspersocket * ($s - $sockspersq * floor($s / $sockspersq)); $url = SquidUpdate::expand($urlArr[$urindex]); $msg = 'PURGE ' . $url . " HTTP/1.0\r\n" . "Connection: Keep-Alive\r\n\r\n"; #$this->debug($msg); @fputs($sockets[$s], $msg); #$this->debug("\n"); } } } #$this->debug("Reading response..."); foreach ($sockets as $socket) { $res = ''; $esc = 0; while (strlen($res) < 100 && $esc < 200) { $res .= @fread($socket, 1024); $esc++; usleep(20); } @fclose($socket); } #$this->debug("\n"); wfProfileOut($fname); }
static function purge($urlArr) { global $wgSquidServers, $wgHTCPMulticastAddress, $wgHTCPPort; self::logit('urls: ' . preg_replace('@\\s+@', ' ', print_r($urlArr, true))); /*if ( (@$wgSquidServers[0]) == 'echo' ) { echo implode("<br />\n", $urlArr) . "<br />\n"; return; }*/ if (empty($urlArr)) { return; } // hack - ignore the stupid user kudos pages that take up so much time! $good = array(); foreach ($urlArr as $u) { if (!preg_match("@User_kudos@", $u)) { $good[] = $u; } } $urlArr = $good; // end hack if ($wgHTCPMulticastAddress && $wgHTCPPort) { return SquidUpdate::HTCPPurge($urlArr); } $fname = 'SquidUpdate::purge'; wfProfileIn($fname); //wfDebug(date("r") . wfBacktrace()); wfDebug(date("r") . " PURGE starting " . print_r($urlArr, true) . "\n"); $maxsocketspersquid = 8; // socket cap per Squid $urlspersocket = 400; // 400 seems to be a good tradeoff, opening a socket takes a while $firsturl = SquidUpdate::expand($urlArr[0]); unset($urlArr[0]); $urlArr = array_values($urlArr); $sockspersq = max(ceil(count($urlArr) / $urlspersocket), 1); if ($sockspersq == 1) { /* the most common case */ $urlspersocket = count($urlArr); wfDebug(date("r") . " PURGE setting urlspersocket to the count of urlArr\n"); } elseif ($sockspersq > $maxsocketspersquid) { $urlspersocket = ceil(count($urlArr) / $maxsocketspersquid); $sockspersq = $maxsocketspersquid; wfDebug(date("r") . " PURGE setting urlspersocket to the ceil " . count($urlArr) . " and {$maxsocketspersquid}\n"); } $totalsockets = count($wgSquidServers) * $sockspersq; $sockets = array(); $isFastlyPurge = array(); /* this sets up the sockets and tests the first socket for each server. */ for ($ss = 0; $ss < count($wgSquidServers); $ss++) { $failed = false; $so = 0; while ($so < $sockspersq && !$failed) { if ($so == 0) { /* first socket for this server, do the tests */ @(list($server, $port) = explode(':', $wgSquidServers[$ss])); if (!isset($port)) { $port = 80; } $isFastly = $server == WH_FASTLY_API_SERVER; #$this->debug("Opening socket to $server:$port"); $error = $errstr = false; $socket = @fsockopen($server, $port, $error, $errstr, 3); #$this->debug("\n"); if (!$socket) { $failed = true; $totalsockets -= $sockspersq; } else { // tmp testing by reuben -- june 7 2013. see purge below too. this may // become permanent if (false || !preg_match('@^https?://([^/]+)(/.*)$@', $firsturl, $m)) { $msg = "PURGE {$firsturl} HTTP/1.0\r\n" . "Connection: Keep-Alive\r\n\r\n"; } else { $host = $m[1]; $loc = $m[2]; $msg = "PURGE {$loc} HTTP/1.0\r\n" . "Host: {$host}\r\n" . "Accept: */*\r\n" . "Connection: Keep-Alive\r\n\r\n"; } self::logit('req (' . $wgSquidServers[$ss] . ', ' . $ss . ') : ' . preg_replace('@\\s+@m', ' ', $msg)); wfDebug(date('r') . " PURGE -- first request {$firsturl}\n"); wfDebug(date('r') . " req headers: " . $msg); @fputs($socket, $msg); #$this->debug("..."); $res = @fread($socket, 512); #$this->debug("\n"); /* Squid only returns http headers with 200 or 404 status, if there's more returned something's wrong Check separately for Varnish and Fastly responses b/c each is different. */ wfDebug(date('r') . " PURGE result: " . print_r($res, true) . "\n"); $isVarnishOK = strpos($res, '200 Purged') !== false; $isFastlyOK = strpos($res, '"status": "ok"') !== false; if (!$isVarnishOK && !$isFastlyOK) { fclose($socket); $failed = true; $totalsockets -= $sockspersq; } else { @stream_set_blocking($socket, false); $sockets[] = $socket; $isFastlyPurge[] = $isFastly; } } } else { /* open the remaining sockets for this server */ list($server, $port) = explode(':', $wgSquidServers[$ss]); if (!isset($port)) { $port = 80; } $isFastlyPurge[$so + 1] = $server == WH_FASTLY_API_SERVER; $sockets[$so + 1] = @fsockopen($server, $port, $error, $errstr, 2); @stream_set_blocking($sockets[$so + 1], false); } $so++; } } wfDebug(date("r") . " PURGE about to do the heavy lifting urlspersocket {$urlspersocket}\n"); if ($urlspersocket > 0) { /* now do the heavy lifting. The fread() relies on Squid returning only the headers */ for ($r = 0; $r < $urlspersocket; $r++) { for ($s = 0; $s < $totalsockets; $s++) { if ($r != 0) { $res = ''; $esc = 0; while (strlen($res) < 100 && $esc < 200) { $res .= @fread($sockets[$s], 512); $esc++; usleep(20); } } $urindex = $r + $urlspersocket * ($s - $sockspersq * floor($s / $sockspersq)); $url = SquidUpdate::expand($urlArr[$urindex]); // tmp testing by reuben -- this may become permanent if (false || !preg_match('@^https?://([^/]+)(/.*)$@', $url, $m)) { $msg = "PURGE {$url} HTTP/1.0\r\n" . "Connection: Keep-Alive\r\n\r\n"; } else { $host = $m[1]; $loc = $m[2]; $msg = "PURGE {$loc} HTTP/1.0\r\n" . "Host: {$host}\r\n" . "Accept: */*\r\n" . "Connection: Keep-Alive\r\n\r\n"; } self::logit('req (' . $s . ') : ' . preg_replace('@\\s+@m', ' ', $msg)); wfDebug(date('r') . " PURGE -- request {$url}\n"); wfDebug(date('r') . " req headers: " . $msg); #$this->debug($msg); @fputs($sockets[$s], $msg); #$this->debug("\n"); } } } #$this->debug("Reading response..."); foreach ($sockets as $i => $socket) { $res = ''; $esc = 0; while (strlen($res) < 100 && $esc < 200) { $res .= @fread($socket, 1024); $esc++; usleep(20); } self::logit('resp (' . $i . ') : ' . preg_replace('@\\s+@m', ' ', $res)); wfDebug(date('r') . " PURGE response: {$res}\n"); @fclose($socket); } #$this->debug("\n"); wfProfileOut($fname); }