forked from cyberpower678/Cyberbot_II
/
externallinksfull.php
178 lines (161 loc) · 6.53 KB
/
externallinksfull.php
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
<?php
ini_set('memory_limit','2G');
echo "----------STARTING UP SCRIPT----------\nStart Timestamp: ".date('r')."\n\n";
require_once('/home/cyberpower678/Peachy/Init.php' );
require_once('/home/cyberpower678/database.inc');
$site2 = Peachy::newWiki( "meta" );
$site = Peachy::newWiki( "cyberbotii" );
$a=0;
//recovery from a crash or a stop midscan
$status = unserialize( file_get_contents( '/home/cyberpower678/bots/cyberbotii/spambotdata/fsbstatus' ) );
if( isset( $status['fstatus'] ) && $status['fstatus'] == 'scan' ) {
preg_match( '/\((.*?) of/i', $status['fscanprogress'], $previousoffset );
$a = $status['fbladd'];
$previousoffset = $previousoffset[1];
}
echo "----------RUN TIMESTAMP: ".date('r')."----------\n\n";
echo "Retrieving blacklists...\n\n";
$status = array( 'fstatus' => 'scan', 'fbladd'=>$a, 'fscanprogress'=>'x' );
updateStatus();
$globalblacklistregex = $site2->initPage( 'Spam blacklist' )->get_text();
$globalblacklistregexarray = explode( "\n", $globalblacklistregex );
$blacklistregex = $site->initPage( 'MediaWiki:Spam-blacklist' )->get_text();
$blacklistregexarray = explode( "\n", $blacklistregex );
$blacklistregex = buildSafeRegexes(array_merge($blacklistregexarray, $globalblacklistregexarray));
$whitelistregex = $site->initPage( 'MediaWiki:Spam-whitelist' )->get_text();
$whitelistregexarray = explode( "\n", $whitelistregex );
$whitelistregex = buildSafeRegexes($whitelistregexarray);
$dblocal = mysqli_connect( 'tools-db', $toolserver_username, $toolserver_password, 's51059__cyberbot' );
$dbwiki = mysqli_connect( 'enwiki.labsdb', $toolserver_username, $toolserver_password, 'enwiki_p' );
$res = mysqli_query( $dbwiki, "SELECT COUNT(*) AS count FROM externallinks;" );
$linkcount = mysqli_fetch_assoc( $res );
$linkcount = $linkcount['count'];
mysqli_free_result( $res );
$offset = 0;
if( isset( $previousoffset ) ) {
$offset = $previousoffset;
unset( $previousoffset );
}
$completed = 0;
$completed = ($offset/$linkcount)*100;
//compile the pages containing blacklisted URLs
echo "Scanning {$linkcount} externallinks in the database...\n\n";
$status = array( 'fstatus' => 'scan', 'fbladd'=>$a, 'fscanprogress'=>round($completed, 3)."% ($offset of {$linkcount})" );
updateStatus();
$starttime = time();
while( $offset < $linkcount ) {
while ( !($res = mysqli_query( $dbwiki, "SELECT * FROM externallinks LIMIT $offset,15000;" )) ) {
echo "Reconnecting to enwiki DB...\n\n";
mysqli_close( $dbwiki );
$dbwiki = mysqli_connect( 'enwiki.labsdb', $toolserver_username, $toolserver_password, 'enwiki_p' );
}
while( $page = mysqli_fetch_assoc( $res ) ) {
if( regexscan( $page['el_to'] ) ) {
while ( !(mysqli_query( $dblocal, "INSERT INTO blacklisted_links (`url`,`page`) VALUES ('".mysqli_escape_string($dblocal, $page['el_to'])."','".mysqli_escape_string($dblocal, $page['el_from'])."');" )) && mysqli_errno( $dblocal ) != 1062 ) {
echo "Attempted INSERT INTO blacklisted_links (`url`,`page`) VALUES ('".mysqli_escape_string($dblocal, $page['el_to'])."','".mysqli_escape_string($dblocal, $page['el_from'])."'); with error ".mysqli_errno( $dblocal )."\n\n";
echo "Reconnecting to local DB...\n\n";
mysqli_close( $dblocal );
$dblocal = mysqli_connect( 'tools-db', $toolserver_username, $toolserver_password, 's51059__cyberbot' );
}
$a++;
}
}
mysqli_free_result( $res );
$offset+=15000;
$completed = ($offset/$linkcount)*100;
$completedin = (((time() - $starttime)*100)/$completed)-(time() - $starttime);
$completedby = time() + $completedin;
$status = array( 'fstatus' => 'scan', 'fbladd'=>$a, 'fscanprogress'=>round($completed, 3)."% ($offset of {$linkcount})", 'fscaneta'=>round($completedby, 0) );
updateStatus();
}
$status = array( 'fstatus' => 'idle' );
updateStatus();
echo "Added $a pages to the local database!\n";
mysqli_close( $dblocal );
mysqli_close( $dbwiki );
//This scans the links with the regexes on the blacklist. If it finds a match, it scans the whitelist to see if it should be ignored.
function regexscan( $link ) {
global $blacklistregex, $whitelistregex;
foreach( $blacklistregex as $regex ) {
if( preg_match($regex, $link) ) {
foreach( $whitelistregex as $wregex ) if( preg_match($wregex, $link) ) return false;
return true;
}
}
return false;
}
//generate a status file
function updateStatus() {
global $status;
return file_put_contents( '/home/cyberpower678/bots/cyberbotii/spambotdata/fsbstatus', serialize($status) );
}
//This is the spam blacklist engine used in MediaWiki adapted for this script. This ensures consistency with the actual wiki.
function stripLines( $lines ) {
return array_filter(
array_map( 'trim',
preg_replace( '/#.*$/', '',
$lines ) ) );
}
function buildSafeRegexes( $lines ) {
$lines = stripLines( $lines );
$regexes = buildRegexes( $lines );
if( validateRegexes( $regexes ) ) {
return $regexes;
} else {
// _Something_ broke... rebuild line-by-line; it'll be
// slower if there's a lot of blacklist lines, but one
// broken line won't take out hundreds of its brothers.
return buildRegexes( $lines, 0 );
}
}
function buildRegexes( $lines, $batchSize=4096 ) {
# Make regex
# It's faster using the S modifier even though it will usually only be run once
//$regex = 'https?://+[a-z0-9_\-.]*(' . implode( '|', $lines ) . ')';
//return '/' . str_replace( '/', '\/', preg_replace('|\\\*/|', '/', $regex) ) . '/Sim';
$regexes = array();
$regexStart = '/(?:https?:)?\/\/+[a-z0-9_\-.]*(';
$regexEnd = ')'.getRegexEnd( $batchSize );
$build = false;
foreach( $lines as $line ) {
if( substr( $line, -1, 1 ) == "\\" ) {
// Final \ will break silently on the batched regexes.
// Skip it here to avoid breaking the next line;
// warnings from getBadLines() will still trigger on
// edit to keep new ones from floating in.
continue;
}
// FIXME: not very robust size check, but should work. :)
if( $build === false ) {
$build = $line;
} elseif( strlen( $build ) + strlen( $line ) > $batchSize ) {
$regexes[] = $regexStart .
str_replace( '/', '\/', preg_replace('|\\\*/|u', '/', $build) ) .
$regexEnd;
$build = $line;
} else {
$build .= '|';
$build .= $line;
}
}
if( $build !== false ) {
$regexes[] = $regexStart .
str_replace( '/', '\/', preg_replace('|\\\*/|u', '/', $build) ) .
$regexEnd;
}
return $regexes;
}
function getRegexEnd( $batchSize ) {
return ($batchSize > 0 ) ? '/Sim' : '/im';
}
function validateRegexes( $regexes ) {
foreach( $regexes as $regex ) {
//wfSuppressWarnings();
$ok = preg_match( $regex, '' );
//wfRestoreWarnings();
if( $ok === false ) {
return false;
}
}
return true;
}