if ($listnum > count($ListArray)) {
$ListEnd = 1;
} else {
$strListUrl = $ListArray[$listnum];
}
break;
}
echo "采集中... 第" . ($listnum + 1) . "页， 成功" . $cg . "条，失败" . $sb . "条";
switch ($ListEnd) {
case 1:
if ($action != "pl") {
dBreakpoint("../../upload/artbreakpoint");
showmsg("<font color='red'><b>采集完成</b></font>", "collect_art_manage.php");
} else {
if ($num >= $arrcount) {
dBreakpoint("../../upload/artbreakpoint");
showmsg("<font color='red'><b>批量采集完成</b></font>", "collect_art_manage.php");
} else {
echo "<br> 此项目数据采集完毕 --- 暂停1秒后继续采集<a href=\"?action=collect_art_cj.php?p_id=" . $p_ids . "&listnum=0&num=" . ($num + 1) . "&action=" . $action . "\">点击采集下一页</a>";
echo "<script language=\"javascript\">setTimeout(\"makeNextPage();\",2000);function makeNextPage(){location.href='collect_art_cj.php?p_id=" . $p_ids . "&listnum=0&num=" . ($num + 1) . "&action=" . $action . "';}</script>";
}
}
break;
default:
cjList();
}
function getUsers()
{
global $db;
$rs = $db->query("select uid,username from discuzcommon_member where uid !=1");
$users = array();
if ($listnum > count($ListArray)) {
$ListEnd = 1;
} else {
$strListUrl = $ListArray[$listnum];
}
break;
}
echo "采集中... 第" . ($listnum + 1) . "页， 成功" . $cg . "条，失败" . $sb . "条 </br>";
switch ($ListEnd) {
case 1:
if ($action != "pl") {
dBreakpoint("../../upload/vodbreakpoint");
showmsg("采集完成", "collect_vod_manage.php");
} else {
if ($num >= $arrcount) {
dBreakpoint("../../upload/vodbreakpoint");
showmsg("批量采集完成", "collect_vod_manage.php");
} else {
echo "此数据采集完毕 --- 暂停3秒后继续采集<script language=\"javascript\">setTimeout(\"makeNextPage();\",2000);function makeNextPage(){location.href='collect_vod_cj.php?p_id=" . $p_ids . "&listnum=0&num=" . ($num + 1) . "&action=" . $action . "';}</script>";
}
}
break;
default:
writetofile("crawel_info.log", $p_id . '{=====}' . $strListUrl . "{=====}start");
cjList();
writetofile("crawel_info.log", $p_id . '{=====}' . $strListUrl . "{=====}end");
break;
}
}
}
}
function insertdata($url, $cjtype)
{
global $db, $action, $rtype, $rpage, $rkey, $rday, $cjurl, $xt, $rid, $cjflag, $backurl;
wBreakpoint("../upload/maccjbreakpoint", "admin_maccj.php?action=" . $action . "&xt=" . $xt . "&rid=" . $rid . "&cjflag=" . $cjflag . "&rpage=" . $rpage . "&rtype=" . $rtype . "&rkey=" . $rkey . "&cjurl=" . $cjurl);
if ($xt == "1") {
$xn_vod = '/<video><last>([\\s\\S]*?)<\\/last><id>([0-9]+)<\\/id><tid>([0-9]+)<\\/tid><name><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/name><type>([\\s\\S]*?)<\\/type><pic>([\\s\\S]*?)<\\/pic><lang>([\\s\\S]*?)<\\/lang><area>([\\s\\S]*?)<\\/area><year>([\\s\\S]*?)<\\/year><state>([\\s\\S]*?)<\\/state><note><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/note><actor><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/actor><director><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/director><dl>([\\s\\S]*?)<\\/dl><des><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/des>([\\s\\S]*?)<\\/video>/';
$xn_url = '/<dd flag="([\\s\\S]*?)"><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/dd>/';
$xn_vod_time = 1;
$xn_vod_id = 2;
$xn_vod_name = 4;
$xn_vod_type = 3;
$xn_vod_pic = 6;
$xn_vod_language = 7;
$xn_vod_area = 8;
$xn_vod_year = 9;
$xn_vod_state = 10;
$xn_vod_remarks = 11;
$xn_vod_starring = 12;
$xn_vod_directed = 13;
$xn_vod_urls = 14;
$xn_vod_des = 15;
} else {
if ($xt == "2") {
$xn_vod = '/<video><last>([\\s\\S]*?)<\\/last><id>([0-9]+)<\\/id><tid>([0-9]+)<\\/tid><name><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/name><type>([\\s\\S]*?)<\\/type><dt>([\\s\\S]*?)<\\/dt><pic>([\\s\\S]*?)<\\/pic><lang>([\\s\\S]*?)<\\/lang><area>([\\s\\S]*?)<\\/area><year>([\\s\\S]*?)<\\/year><state>([\\s\\S]*?)<\\/state><note><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/note><actor><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/actor><director><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/director><dl>([\\s\\S]*?)<\\/dl><des><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/des><reurl>([\\s\\S]*?)<\\/reurl><\\/video>/';
$xn_url = '/<dd flag="([\\s\\S]*?)"><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/dd>/';
$xn_vod_time = 1;
$xn_vod_id = 2;
$xn_vod_name = 4;
$xn_vod_type = 3;
$xn_vod_pic = 7;
$xn_vod_language = 8;
$xn_vod_area = 9;
$xn_vod_year = 10;
$xn_vod_state = 11;
$xn_vod_remarks = 12;
$xn_vod_starring = 13;
$xn_vod_directed = 14;
$xn_vod_urls = 15;
$xn_vod_des = 16;
} else {
$xn_vod = '/<vod><id>([0-9]+)<\\/id><name><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/name><note><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/note><state>([\\s\\S]*?)<\\/state><type>([\\s\\S]*?)<\\/type><starring><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/starring><directed><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/directed><pic>([\\s\\S]*?)<\\/pic><time>([\\s\\S]*?)<\\/time><year>([\\s\\S]*?)<\\/year><area><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/area><language><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/language><urls>([\\s\\S]*?)<\\/urls><des><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/des><\\/vod>/';
$xn_url = '/<url from="([\\s\\S]*?)"><\\!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/url>/';
$xn_vod_id = 1;
$xn_vod_name = 2;
$xn_vod_remarks = 3;
$xn_vod_state = 4;
$xn_vod_type = 5;
$xn_vod_starring = 6;
$xn_vod_directed = 7;
$xn_vod_pic = 8;
$xn_vod_time = 9;
$xn_vod_year = 10;
$xn_vod_area = 11;
$xn_vod_language = 12;
$xn_vod_des = 14;
$xn_vod_urls = 13;
}
}
$html = getPage($url, "utf-8");
if (html == false) {
echo "<table class=\"tb\"><tr><td colspan=2> <a href=\"javascript:void(0)\" onclick=\"location.reload();\">获取数据失败，请点击我重试</a></td></tr></table>";
exit;
}
if ($xt == "1" || $xt == "2") {
$pagecount = intval(getBody($html, "pagecount=\"", "\" pagesize"));
} else {
$pagecount = intval(getBody($html, "<pagecount>", "</pagecount>"));
}
if ($pagecount == 0) {
echo "<table class=\"tb\"><tr><td><br>没有任何可用数据<script language=\"javascript\">setTimeout(\"gonextpage();\",5000);function gonextpage(){location.href='?action=list&xt=" . $xt . "&rid=" . $rid . "&cjflag=" . $cjflag . "&cjurl=" . $cjurl . "';}</script></td></tr></table>";
exit;
}
echo "<table class=\"tb\"><tr><td colspan=2>视频采集地址 " . $url . "</td></tr><tr><td colspan=2> 共" . $pagecount . "页，正在采集第" . $rpage . "页</td></tr>";
preg_match_all($xn_vod, $html, $array3);
$i = 0;
foreach ($array3[1] as $key => $value) {
$rc = false;
$vodid = $array3[$xn_vod_id][$key];
$vodname = $array3[$xn_vod_name][$key];
$vodremarks = $array3[$xn_vod_remarks][$key];
$vodstate = $array3[$xn_vod_state][$key];
$vodtype = $cjflag . $array3[$xn_vod_type][$key];
$vodstarring = $array3[$xn_vod_starring][$key];
$voddirected = $array3[$xn_vod_directed][$key];
$vodpic = $array3[$xn_vod_pic][$key];
$vodtime = $array3[$xn_vod_time][$key];
$vodyear = $array3[$xn_vod_year][$key];
$vodarea = $array3[$xn_vod_area][$key];
$vodlanguage = $array3[$xn_vod_language][$key];
$voddes = $array3[$xn_vod_des][$key];
$vodurls = $array3[$xn_vod_urls][$key];
preg_match_all($xn_url, $vodurls, $array4);
$vodsubname = "";
$vodcolor = "";
$vodhitstime = "";
$voddownurl = "";
$vodplayserver = "0";
$vodfrom = "";
$vodurl = "";
$vodname = replaceStr($vodname, "'", "");
if (strlen($vodname) > 255) {
$vodname = substring($vodname, 255);
}
$vodenname = Hanzi2PinYin($vodname);
if (strlen($vodenname) > 255) {
$vodenname = substring($vodenname, 255);
}
if (!isN($vodenname)) {
$vodletter = strtoupper(substring($vodenname, 1));
}
$vodstate = replaceStr($vodstate, "[", "");
$vodstate = replaceStr($vodstate, "]", "");
if (!isNum($vodstate)) {
$vodstate = 0;
}
$vodstarring = replaceStr($vodstarring, "'", "");
$vodstarring = replaceStr($vodstarring, "、", " ");
$vodstarring = replaceStr($vodstarring, ",", " ");
$vodstarring = replaceStr($vodstarring, "，", " ");
$vodstarring = replaceStr($vodstarring, " ", " ");
if (strlen($vodstarring) > 255) {
$vodstarring = substring($vodstarring, 255);
}
$vodyear = replaceStr($vodyear, "'", "");
if (strlen($vodyear) > 32) {
$vodyear = substring($vodyear, 32);
}
$vodlanguage = replaceStr($vodlanguage, "'", "");
if (strlen($vodlanguage) > 32) {
$vodlanguage = substring($vodlanguage, 32);
}
$vodarea = replaceStr($vodarea, "'", "");
if (strlen($vodarea) > 32) {
$vodarea = substring($vodarea, 32);
}
$vodpic = replaceStr($vodpic, "'", "");
if (strlen($vodpic) > 255) {
$vodpic = substring($vodpic, 255);
}
$voddes = htmlDecode($voddes);
$voddes = replaceStr($voddes, "'", "");
$vodtype = getTypeID($vodtype);
if (!isNum($vodtype)) {
$vodtype = 0;
}
if ($vodtype > 0) {
$sql = "SELECT * FROM {pre}vod WHERE d_name ='" . $vodname . "' ";
$row = $db->getRow($sql);
if (!$row) {
foreach ($array4[1] as $key => $value) {
if ($rc) {
$vodfrom .= "\$\$\$";
$vodurl .= "\$\$\$";
}
$vodfrom .= getFrom($value);
if ($xt == "1") {
$vodurl .= getVUrl($array4[2][$key]);
} else {
$vodurl .= $array4[2][$key];
}
if (substring($vodurl, 1, strlen($vodurl) - 1) == Chr(13)) {
$vodurl = substring($vodurl, strlen($vodurl) - 1);
}
$vodurl = replaceStr($vodurl, Chr(10), "#");
$vodurl = replaceStr($vodurl, Chr(13), "#");
$vodurl = replaceStr($vodurl, "##", "#");
$vodurl = replaceStr($vodurl, "'", "''");
$rc = true;
}
$db->Add("{pre}vod", array("d_type", "d_name", "d_subname", "d_enname", "d_letter", "d_state", "d_color", "d_content", "d_remarks", "d_pic", "d_level", "d_hits", "d_starring", "d_directed", "d_year", "d_area", "d_language", "d_addtime", "d_time", "d_playurl", "d_downurl", "d_playfrom", "d_playserver"), array($vodtype, $vodname, $vodsubname, $vodenname, $vodletter, $vodstate, $vodcolor, $voddes, $vodremarks, $vodpic, 0, 0, $vodstarring, $voddirected, $vodyear, $vodarea, $vodlanguage, date('Y-m-d H:i:s', time()), date('Y-m-d H:i:s', time()), $vodurl, $voddownurl, $vodfrom, $vodplayserver));
wTips($vodname, "新增数据");
} else {
$tmpplayurl = $row["d_playurl"];
$tmpplayfrom = $row["d_playfrom"];
$tmpplayserver = $row["d_playserver"];
foreach ($array4[1] as $key => $value) {
$vodfrom = getFrom($value);
if ($xt == "1") {
$vodurl = getVUrl($array4[2][$key]);
} else {
$vodurl = $array4[2][$key];
}
if (substring($vodurl, 1, strlen($vodurl) - 1) == Chr(13)) {
$vodurl = substring($vodurl, strlen($vodurl) - 1);
}
$vodurl = replaceStr($vodurl, Chr(10), "#");
$vodurl = replaceStr($vodurl, Chr(13), "#");
$vodurl = replaceStr($vodurl, "##", "#");
$vodurl = replaceStr($vodurl, "'", "''");
if ($tmpplayurl == $vodurl) {
$resultdes = "无需更新播放地址";
continue;
} else {
if (isN($vodfrom)) {
wTips($vodname, "播放器类型为空，跳过");
continue;
} else {
if (isN($tmpplayurl) || strpos("," . $row["d_playfrom"], $vodfrom) <= 0) {
$resultdes = "新增播放地址组";
$tmpplayurl .= "\$\$\$" . $vodurl;
$tmpplayfrom .= "\$\$\$" . $vodfrom;
$tmpplayserver .= "\$\$\$" . $vodplayserver;
} else {
$resultdes = "更新播放地址";
$arr1 = explode("\$\$\$", $tmpplayurl);
$arr2 = explode("\$\$\$", $tmpplayfrom);
$rc = false;
$tmpplayurl = "";
for ($k = 0; $k < count($arr2); $k++) {
if ($rc) {
$tmpplayurl .= "\$\$\$";
}
if ($arr2[$k] == $vodfrom) {
$arr1[$k] = $vodurl;
}
$tmpplayurl .= $arr1[$k];
$rc = true;
}
}
}
}
}
$tmpplayurl = replaceStr($tmpplayurl, Chr(13), "#");
if (strpos("," . $row["d_pic"], "http:") <= 0) {
$vodpic = $row["d_pic"];
}
$db->Update("{pre}vod", array("d_state", "d_pic", "d_remarks", "d_time", "d_playurl", "d_playfrom", "d_playserver"), array($vodstate, $vodpic, $vodremarks, date('Y-m-d H:i:s', time()), $tmpplayurl, $tmpplayfrom, $tmpplayserver), "d_id=" . $row["d_id"]);
wTips($vodname, $resultdes);
}
unset($row);
} else {
wTips($vodname, "没有绑定分类、跳过");
}
unset($array4);
}
unset($array3);
echo "</table>";
if ($action == "cjday" || $action == "cjall" || $action == "cjtype") {
if ($rpage >= $pagecount) {
dBreakpoint("../upload/maccjbreakpoint");
echo "<br>数据采集完成<script language=\"javascript\">setTimeout(\"gonextpage();\",3000);function gonextpage(){location.href='?action=list&fa=1&xt=" . $xt . "&rid=" . $rid . "&cjflag=" . $cjflag . "&rtype=" . $rtype . "&cjurl=" . $cjurl . "';}</script>";
} else {
echo "<br>暂停3秒后继续采集...<script language=\"javascript\">setTimeout(\"gonextpage();\",3000);function gonextpage(){location.href='?action=" . $action . "&rpage=" . ($rpage + 1) . "&rtype=" . $rtype . "&rday=" . $rday . "&xt=" . $xt . "&rid=" . $rid . "&cjflag=" . $cjflag . "&cjurl=" . $cjurl . "';}</script><a href=\"?action=" . $action . "&rpage=" . ($rpage + 1) . "&rtype=" . $rtype . "&xt=" . $xt . "&rid=" . $rid . "&cjflag=" . $cjflag . "&cjurl=" . $cjurl . "\" >点击进入下一页</a>";
}
} else {
dBreakpoint("../upload/maccjbreakpoint");
echo "<br>数据采集完成<script language=\"javascript\">setTimeout(\"gonextpage();\",3000);function gonextpage(){location.href='?action=list&fa=1&xt=" . $xt . "&rid=" . $rid . "&cjflag=" . $cjflag . "&rpage=" . $rpage . "&rtype=" . $rtype . "&cjurl=" . $cjurl . "';}</script>";
}
}
