function getInfo($path, $offset, $length)
{
$retult = uncompress($path);
return substr($retult, $offset, $length);
}
function uncompress_file()
{
$tpl = new templates();
@chmod("ressources/logs", 0777);
$ID = $_GET["uncompress"];
if (!is_numeric($ID)) {
die;
}
$t = $_GET["t"];
$sock = new sockets();
$q = new mysql_squid_builder();
$mydir = dirname(__FILE__);
$ligne = mysql_fetch_array($q->QUERY_SQL("SELECT filename,fileext FROM squid_storelogs WHERE ID = '{$ID}'"));
$filename = $ligne["filename"];
$ttr = explode(".", $ligne["filename"]);
$uncompressedfile = "ressources/logs/{$ttr[0]}.{$ligne["fileext"]}";
$filepath = "{$mydir}/ressources/logs/{$filename}";
writelogs("uncompress filename ID:{$_GET["uncompress"]}", __FUNCTION__, __FILE__, __LINE__);
$sql = "SELECT filecontent INTO DUMPFILE '{$filepath}' FROM squid_storelogs WHERE ID = '{$ID}'";
$ligne = mysql_fetch_array($q->QUERY_SQL("{$sql}"));
if (!$q->ok) {
echo $tpl->_ENGINE_parse_body("<H1 style='color:#d32d2d;background:none'>{failed} {$q->mysql_error}</H1>");
}
if (!uncompress($filepath, $uncompressedfile)) {
@unlink($filepath);
echo $tpl->_ENGINE_parse_body("<H1 style='color:#d32d2d;background:none'>{failed}</H1>");
return;
}
@unlink($filepath);
echo "<script>UnCompressCheck{$t}(1)</script>";
}
include_once('ressources/class.templates.inc');
include_once('ressources/class.ldap.inc');
include_once('ressources/class.users.menus.inc');
include_once('ressources/class.squid.inc');
$usersmenus=new usersMenus();
if(!$usersmenus->AsSquidAdministrator){
$tpl=new templates();
$alert=$tpl->_ENGINE_parse_body('{ERROR_NO_PRIVS}');
echo "alert('$alert');";
die();
}
if( isset($_GET['TargetArticaUploaded']) ){upload_artica_perform();exit();}
if(isset($_GET["file-uploader-demo1"])){upload_artica_final();exit;}
if(isset($_GET["uncompress"])){uncompress();exit;}
if(isset($_GET["remove"])){remove();exit;}
if(isset($_GET["tabs"])){tabs();exit;}
if(isset($_GET["popup"])){popup();exit;}
if(isset($_GET["manu"])){manu();exit;}
if(isset($_GET["compile-list"])){squid_compile_list();exit;}
js();
function js(){
$page=CurrentPageName();
$tpl=new templates();
header("content-type: application/x-javascript");
$title=$tpl->_ENGINE_parse_body("{compilation_status}");
$html="YahooWin4('585','$page?tabs=yes','$title');";
die;
}
if (isset($_GET["manual-update"])) {
manual_update();
exit;
}
if (isset($_GET['TargetArticaUploaded'])) {
upload_artica_perform();
exit;
}
if (isset($_GET["file-uploader-demo1"])) {
upload_artica_final();
exit;
}
if (isset($_GET["uncompress"])) {
uncompress();
exit;
}
if (isset($_GET["remove"])) {
remove();
exit;
}
js();
function js()
{
header("content-type: application/x-javascript");
$tpl = new templates();
$page = CurrentPageName();
$title = $tpl->javascript_parse_text("{upload_a_pdf_file}");
echo "YahooWinBrowse('700','{$page}?manual-update=yes&ID={$_GET["ID"]}','{$title}',true)";
}
function import(){
echo "Starting......: [DOWNLOAD]:: Artica database community Importing categories\n";
$sock=new sockets();
$unix=new unix();
$FilterCommunityMD5=unserialize(base64_decode($sock->GET_INFO("FilterCommunityMD5")));
$TmpfileDataMD5=$unix->FILE_TEMP();
$t1=time();
$q=new mysql();
$rownum=$q->COUNT_ROWS("dansguardian_community_categories","artica_backup");
echo "Starting......: [DOWNLOAD]:: Artica database community (current $rownum rows)\n";
$fp = fopen ($TmpfileDataMD5, 'w+');//This is the file where we save the information
$ch = curl_init('http://www.artica.fr/tmp/open-webfilter.gz.md5');//Here is the file we are downloading
curl_setopt($ch, CURLOPT_TIMEOUT, 50);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_exec($ch);
curl_close($ch);
fclose($fp);
$remotemd5=unserialize(base64_decode(@file_get_contents($TmpfileDataMD5)));
for($i=0;$i<count($remotemd5)+1;$i++){
$tmpfile=$unix->FILE_TEMP().".$i.gz";
$d1=trim($FilterCommunityMD5["open-webfilter2.$i.gz"]);
$d2=trim($remotemd5["open-webfilter2.$i.gz"]);
if($d2==null){
echo "Starting......: [DOWNLOAD]:: [$i] open-webfilter2.$i.gz MD5 is null aborting\n";
continue;
}
if($rownum>0){
echo "Starting......: [DOWNLOAD]:: [$i] Artica database open-webfilter2.$i.gz [$d1] [$d2]\n";
if($d1==$d2){
echo "Starting......: [DOWNLOAD]:: [$i] Artica database open-webfilter2.$i.gz (unchanged)\n";
continue;
}
}
$fp = fopen ($tmpfile, 'w+');//This is the file where we save the information
echo "Starting......: [DOWNLOAD]:: Artica database downloading file N.$i....\n";
$ch = curl_init("http://www.artica.fr/tmp/open-webfilter2.$i.gz");//Here is the file we are downloading
WriteMyLogs("http://www.artica.fr/tmp/open-webfilter2.$i.gz d1:$d1 d2:$d2",__FUNCTION__,__FILE__,__LINE__);
curl_setopt($ch, CURLOPT_TIMEOUT, 50);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_exec($ch);
curl_close($ch);
fclose($fp);
$size=filesize($tmpfile)/1024;
WriteMyLogs("[$i] Artica database community uncompress database ($size)Ko",__FUNCTION__,__FILE__,__LINE__);
echo "Starting......: [DOWNLOAD]:: [$i] Artica database community uncompress database ($size)Ko\n";
uncompress($tmpfile,"/tmp/FilterCommunity.$i.sql");
@unlink($tmpfile);
if(!is_file("/tmp/FilterCommunity.$i.sql")){
echo "Starting......: [DOWNLOAD]:: [$i] Unable to stat /tmp/FilterCommunity.$i.sql aborting\n";
continue;
}
if(filesize("/tmp/FilterCommunity.$i.sql")<600){
WriteMyLogs("FilterCommunity.$i.sql <600 aborting",__FUNCTION__,__FILE__,__LINE__);
continue;
}
$GLOBALS["NEWFILES"][]=basename($tmpfile)." -> ". round($size/1000)." Mo";
echo "Starting......: [DOWNLOAD]:: [$i] Artica database community file $i ". filesize("/tmp/FilterCommunity.$i.sql") ." bytes\n";
if(ParseGzSqlFile("/tmp/FilterCommunity.$i.sql")){
$NewFilterCommunityMD5["open-webfilter2.$i.gz"]=$d2;
$sock->SET_INFO("FilterCommunityMD5",base64_encode(serialize($NewFilterCommunityMD5)));
}else{
WriteMyLogs("/tmp/FilterCommunity.$i.gz failed to extract... ",__FUNCTION__,__FILE__,__LINE__);
}
}
$newrownum=$q->COUNT_ROWS("dansguardian_community_categories","artica_backup");
echo "Starting......: Artica database community (now is $newrownum rows)\n";
WriteMyLogs("Artica database community (now is $newrownum rows)",__FUNCTION__,__FILE__,__LINE__);
$sock->SET_INFO("FilterCommunityMD5",base64_encode(serialize($NewFilterCommunityMD5)));
$t2=time();
$final_rows=$newrownum-$rownum;
if($final_rows>0){
$time_duration=distanceOfTimeInWords($t1,$t2);
$unix->send_email_events("Web $final_rows categorized websites $time_duration", @implode("\n",$GLOBALS["NEWFILES"]),"proxy");
$q=new mysql();
$q->QUERY_SQL("OPTIMIZE table dansguardian_community_categories","artica_backup");
}
WriteCategoriesStatus(true);
$unix->THREAD_COMMAND_SET("/usr/share/artica-postfix/bin/artica-update --squidguard");
}
/**
* Computes datas for all subPeriodValues values
* and saves it in a new row in the database table archives
*
* @return void
*/
function compute()
{
if (CURRENT_PERIOD_SIMPLE_ARCHIVE && $this->state === DB_ARCHIVES_TEMP) {
$this->toRecord['simple'] = 1;
} else {
$this->toRecord['simple'] = 0;
}
// date in the IN(...) SQL field for unique visitors
$inIdArchives = '';
$this->toRecord['vis_pag_grp'] = array();
// init
$toInitInt = $this->intValuesToSum;
$toInitArray = array_merge($this->arrayOneDimToSum, $this->arrayPmvSumToSum, $this->arrayIntToSum);
foreach ($toInitInt as $value) {
$this->toRecord[$value] = 0;
}
foreach ($toInitArray as $value) {
$this->toRecord[$value] = array();
}
// for each subPeriod, sum all values
foreach ($this->subPeriodValues as $idArchives => $subPeriodValues) {
foreach ($this->intValuesToSum as $name) {
$this->toRecord[$name] += $subPeriodValues[$name];
}
// case we don't want a SIMPLE archiving
if ($this->toRecord['simple'] == 0) {
//printDebug("Sum2 : <br>");
foreach ($this->arrayOneDimToSum as $name) {
if (!isset($subPeriodValues[$name])) {
$subPeriodValues[$name] = compress(serialize(array()), $subPeriodValues['compressed']);
}
$this->toRecord[$name] = getArrayOneDimVeryVeryVerySpecialSum($this->toRecord[$name], unserialize(uncompress($subPeriodValues[$name], $subPeriodValues['compressed'])), "sum");
}
//printDebug("Sum3 : <br>");
foreach ($this->arrayPmvSumToSum as $name) {
if (!isset($subPeriodValues[$name])) {
$subPeriodValues[$name] = compress(serialize(array()), $subPeriodValues['compressed']);
}
$this->toRecord[$name] = getArrayOneDimVeryVeryVerySpecialSum($this->toRecord[$name], unserialize(uncompress($subPeriodValues[$name], $subPeriodValues['compressed'])), "sumArray");
}
//printDebug("Sum4 : <br>");
foreach ($this->arrayIntToSum as $name) {
if (!isset($subPeriodValues[$name])) {
$subPeriodValues[$name] = compress(serialize(array()), $subPeriodValues['compressed']);
}
$this->toRecord[$name] = getArrayOneDimVeryVeryVerySpecialSum($this->toRecord[$name], unserialize(uncompress($subPeriodValues[$name], $subPeriodValues['compressed'])), "sumArray");
}
//printDebug("<h1>Sum5 $idArchives: </h1><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br>");
//printDebug("table to add");
//printDebug(unserialize($subPeriodValues['vis_pag_grp']));
if (!is_string($subPeriodValues['vis_pag_grp'])) {
$subPeriodValues['vis_pag_grp'] = compress(serialize(array()), $subPeriodValues['compressed']);
}
$this->toRecord['vis_pag_grp'] = getPagGrpArrayMultiDimSum($this->toRecord['vis_pag_grp'], unserialize(uncompress($subPeriodValues['vis_pag_grp'], $subPeriodValues['compressed'])));
}
$inIdArchives .= "'" . $idArchives . "', ";
}
//printDebug($this->toRecord['vis_pag_grp']);
//exit;
/*
* Sorting arrays
*/
$this->sortAndLimitToRecord();
$inIdArchives = substr($inIdArchives, 0, strlen($inIdArchives) - 2);
// if year, look for info averaging month archives
if ($this->periodType === DB_ARCHIVES_PERIOD_YEAR) {
$r = query("SELECT sum(nb_uniq_vis) as nb_uniq_vis, max(nb_max_pag) as nb_max_pag\n\t\t\t\t\tFROM " . T_ARCHIVES . "\n\t\t\t\t\tWHERE idarchives IN (" . $inIdArchives . ")");
// unique visitors returning sum on the period
$nb_uniq_vis_returning = query("SELECT sum(nb_uniq_vis_returning) as nb_uniq_vis_returning\n\t\t\t\t\t\t\t\t\t\t\tFROM " . T_ARCHIVES . "\n\t\t\t\t\t\t\t\t\t\t\tWHERE idarchives IN (" . $inIdArchives . ")");
// returning visitors
$returning = query("SELECT nb_vis_returning as s, idarchives as pseudodate, \n\t\t\t\t\t\t\t\t\tnb_pag_returning as sp, date1 as pseudodate2\n\t\t\t\t\t\tFROM " . T_ARCHIVES . "\n\t\t\t\t\t\tWHERE idsite = " . $this->site->getId() . "\n\t\t\t\t\t\tAND idarchives IN (" . $inIdArchives . ")\n\t\t\t\t\t\tGROUP BY pseudodate\n\t\t\t\t\t\t");
} else {
$r = query("SELECT count(distinct idcookie) as nb_uniq_vis, max(total_pages) as nb_max_pag\n\t\t\t\tFROM " . T_VISIT . "\n\t\t\t\tWHERE idsite = " . $this->site->getId() . "\n\t\t\t\tAND server_date IN (" . $inIdArchives . ")");
// unique visitors returning sum on the period
$nb_uniq_vis_returning = query("SELECT count(distinct idcookie) as nb_uniq_vis_returning\t\n\t\t\t\t\t\t\t\t\t\t\tFROM " . T_VISIT . "\n\t\t\t\t\t\t\t\t\t\t\tWHERE idsite = " . $this->site->getId() . "\n\t\t\t\t\t\t\t\t\t\t\tAND server_date IN (" . $inIdArchives . ")\n\t\t\t\t\t\t\t\t\t\t\tAND returning = 1\n\t\t\t\t \t\t\t\t\t\t\tGROUP BY returning");
// returning visitors
$returning = query("SELECT count(*) as s, server_date as pseudodate, \n\t\t\t\t\t\t\t\t\t\tsum(total_pages) as sp\n\t\t\t\t\t\tFROM " . T_VISIT . "\n\t\t\t\t\t\tWHERE idsite = " . $this->site->getId() . "\n\t\t\t\t\t\tAND server_date IN (" . $inIdArchives . ")\n\t\t\t\t\t\tAND returning = 1\n\t\t\t\t\t\tGROUP BY pseudodate");
// for each returning visit, how many time did they return?
$f1 = query("SELECT count(*) as s\n\t\t\t\t\t\tFROM " . T_VISIT . "\n\t\t\t\t\t\tWHERE idsite = " . $this->site->getId() . "\n\t\t\t\t\t\tAND server_date IN (" . $inIdArchives . ")\n\t\t\t\t\t\tGROUP BY idcookie");
$res = array();
while ($fx = mysql_fetch_assoc($f1)) {
@$res[(int) $fx['s']]++;
}
ksort($res);
$this->toRecord['vis_nb_vis'] = $res;
}
$nb_uniq_vis_returning = mysql_fetch_assoc($nb_uniq_vis_returning);
$this->toRecord['nb_uniq_vis_returning'] = $nb_uniq_vis_returning['nb_uniq_vis_returning'];
$l = mysql_fetch_assoc($r);
$this->toRecord['nb_uniq_vis'] = $l['nb_uniq_vis'];
$this->toRecord['nb_max_pag'] = $l['nb_max_pag'];
// visits per period, new vs returning, and pages hits
$returningTotal = 0;
$returningPag = 0;
$this->toRecord['vis_period'] = array();
while ($l2 = mysql_fetch_assoc($returning)) {
$visitsTotal = $this->subPeriodValues[$l2['pseudodate']]['nb_vis'];
$pagesTotal = $this->subPeriodValues[$l2['pseudodate']]['nb_pag'];
$returningVisits = $l2['s'];
$returningPag += $l2['sp'];
$returningTotal += $returningVisits;
$this->toRecord['vis_period'][isset($l2['pseudodate2']) ? $l2['pseudodate2'] : $l2['pseudodate']] = array(ARRAY_INDEX_RETURNING_COUNT => (int) $returningVisits, ARRAY_INDEX_NEW_COUNT => $visitsTotal - $returningVisits, ARRAY_INDEX_PAGES_COUNT => (int) $pagesTotal);
}
ksort($this->toRecord['vis_period']);
// number of returning visits
$this->toRecord['nb_vis_returning'] = $returningTotal;
$this->toRecord['nb_pag_returning'] = $returningPag;
//printDebug("TO RECORD<br>");
//printDebug($this->toRecord);
/*
* init
*/
$this->initDb();
/*
* final save, close your eyes and prey
*/
$this->saveDb();
/**
* delete all visit/month records
*/
$this->deleteOldRecords();
}
function uncompress($srcName, $dstName)
{
$string = implode("", gzfile($srcName));
$fp = fopen($dstName, "w");
fwrite($fp, $string, strlen($string));
fclose($fp);
}
$subs_file = file_get_contents("export.txt");
$subs = array();
preg_match_all("/.*[\\n\\r]+/iu", $subs_file, $matches);
foreach ($matches[0] as $line) {
$subtitle = explode("\t", $line);
if (!is_numeric($subtitle[1])) {
continue;
}
uncompress('/home/soshial/subs/opensubtitles/' . $subtitle[1] . '.gz', '/home/soshial/subs/opensubtitles2/' . $subtitle[1] . '.srt');
$sub_text = file_get_contents('/home/soshial/subs/opensubtitles2/' . $subtitle[1] . '.srt');
if (detect_encoding($sub_text) == 'windows-1251') {
$sub_text = strip_tags_smart(iconv('windows-1251', 'UTF-8//IGNORE', $sub_text));
}
// изменяем кодировку на юникод, и вычищаем от тегов
$sub_text = preg_replace('/\\s*\\d+\\r\\n\\d\\d:\\d\\d:\\d\\d,\\d+ --> \\d\\d:\\d\\d:\\d\\d,\\d+/i', '', $sub_text, -1, $count);
// вырезаем временные метки
$sub_text = preg_replace('/\\.{2,}/i', '…', $sub_text, -1, $count);
// заменяем многоточия
/* склеиваем текст */
$sub_text = preg_replace("/([^!…\\?_][a-z0-9а-яё])(\\s)*\\r\\n(\\s)*([…«\"\\-]{0,1}(\\s)*[a-zа-яё]{1,}[^\\)])/u", '$1 $4', $sub_text, -1, $count);
// ориентация на отсутствие препинаний в конце первого параграфа и на строчные в начале второго п.,
// где перед буквами могут находиться кавычки: «" и дефис
// предполагается автоматическое объединение
$sub_text = preg_replace("/([^!…\\?_][a-z0-9А-яё,;—–\\-\\»\\)IVX])(\\s)*\\r\\n(\\s)*([«\\(\\[\"\\-–—=…]{0,1}(\\s)*[a-z0-9а-яё\\?\\!…\"]+[^\\)])/u", '$1 $4', $sub_text, -1, $count);
function import($ou, $path)
{
$usersM = new usersMenus();
$unix = new unix();
if (!is_file($path)) {
echo "{$path}, no such file\n";
die;
}
$ldap = new clladp();
$oudn = "ou={$ou},dc=organizations,{$ldap->suffix}";
if (!$ldap->ExistsDN($oudn)) {
$ldap->AddOrganization($ou);
}
$tmpfile = $unix->FILE_TEMP();
uncompress($path, $tmpfile);
$datas = unserialize(base64_decode(@file_get_contents($tmpfile)));
if (!is_array($datas)) {
if ($GLOBALS["VERBOSE"]) {
echo "Unable to import {$ou} {$path}, no such array\n";
return;
}
$unix->send_email_events(basename(__FILE__) . "::Unable to import {$ou} {$path}, no such array", null, "import");
die;
}
$usersArray = $datas["USERS"];
$groupsArray = $datas["GROUPS"];
unset($datas);
echo "Creating groups in `{$ou}`";
while (list($num, $hash) = each($groupsArray)) {
$ORG_GID = $hash["main_array"]["gid"];
$gp = new groups();
$gpid = $gp->GroupIDFromName($ou, $hash["groupName"]);
if (!is_numeric($gpid)) {
$gp->groupName = $hash["groupName"];
$gp->add_new_group($hash["groupName"], $ou);
$gpid = $gp->GroupIDFromName($ou, $hash["groupName"]);
}
$gp = new groups($gpid);
$members = $hash["members"];
while (list($a, $b) = each($members)) {
echo "Insert {$b} user to {$hash["groupName"]}/{$gpid}\n";
$gp->AddUsertoThisGroup($b);
}
$gp->saveDescription($hash["main_array"]["description"]);
$gp->ArticaGroupPrivileges = $hash["main_array"]["ArticaGroupPrivileges"];
$gp->Privileges_array = $hash["Privileges_array"];
$gp->SavePrivileges();
$GROUPSORGS[$ORG_GID] = $gpid;
}
while (list($num, $hash) = each($usersArray)) {
$array_groups = $hash["array_groups"];
unset($hash["dn"]);
unset($hash["UserExists"]);
unset($hash["ou"]);
unset($hash["local_sid"]);
unset($hash["objectClass_array"]);
unset($hash["group_id"]);
unset($hash["sambaPrimaryGroupSID"]);
unset($hash["accountGroup"]);
unset($hash["uidNumber"]);
unset($hash["sambaSID"]);
unset($hash["sambaPrimaryGroupGID"]);
unset($hash["gidNumber_array"]);
$samba_groups = $hash["samba_groups"];
unset($hash["ldapClass"]);
unset($hash["attributs_array"]);
unset($hash["samba_groups"]);
$users = new user($hash["uid"]);
$users->ou = $ou;
$users->group_id = $GROUPSORGS[$array_groups[0]];
while (list($a, $orgd) = each($hash)) {
$users->a = $orgd;
}
$users->add_user();
if ($usersM->SAMBA_INSTALLED) {
if (!$hash["NotASambaUser"]) {
$users->Samba_edit_user();
}
}
}
}
/**
* Checks an IPv6 address
*
* Checks if the given IP is a valid IPv6 address
*
* @param string $ip An IPv6 address
* @return bool true if $ip is a valid IPv6 address
*/
function check_ipv6($ip)
{
$ip = uncompress($ip);
list($ipv6, $ipv4) = split_v6_v4($ip);
$ipv6 = explode(':', $ipv6);
$ipv4 = explode('.', $ipv4);
if (count($ipv6) === 8 && count($ipv4) === 1 || count($ipv6) === 6 && count($ipv4) === 4) {
foreach ($ipv6 as $ipv6_part) {
// The section can't be empty
if ($ipv6_part === '') {
return false;
}
// Nor can it be over four characters
if (strlen($ipv6_part) > 4) {
return false;
}
// Remove leading zeros (this is safe because of the above)
$ipv6_part = ltrim($ipv6_part, '0');
if ($ipv6_part === '') {
$ipv6_part = '0';
}
// Check the value is valid
$value = hexdec($ipv6_part);
if (dechex($value) !== strtolower($ipv6_part) || $value < 0 || $value > 0xffff) {
return false;
}
}
if (count($ipv4) === 4) {
foreach ($ipv4 as $ipv4_part) {
$value = (int) $ipv4_part;
if ((string) $value !== $ipv4_part || $value < 0 || $value > 0xff) {
return false;
}
}
}
return true;
} else {
return false;
}
}
