function runProducer() { $kmlPath = getconfig('kmlPath'); $xml_file = getFileList($kmlPath); $lockfile = '/tmp/producer.lock'; if (empty($xml_file)) { logs(date('Y-m-d h:i:m') . "XML source files downloaded from the FTP is empty."); unlink($lockfile); exit; } sort($xml_file); $startTime = explode(' ', microtime()); $totalNum = 0; $i = $n = 1; foreach ($xml_file as $f) { //解析文件生成数组 $data = paseXml($f); //XML格式检查 $res = isFormat($data, $f); if ($res === false) { continue; } //格式化 $kmldata = formatKmlData($data, $f); $i++; $fNum = count($kmldata); $totalNum += $fNum; //入队列 $kafkaTime = explode(' ', microtime()); $fileName = basename($f); insertKafka($kmldata, $fileName); logs($i . '>>>' . basename($f) . ',file count:' . $fNum . ',total:' . $totalNum . ',into kafka time:' . getTime($kafkaTime)); if ($n > 100) { usleep(200); $n = 1; } //备份文件: backFile($f); } logs('Total time:' . getTime($startTime)); //unlock(); //$lockfile = '/tmp/producer.lock'; unlink($lockfile); exit; }
function isFormat($data, $f) { $res = true; $f = basename($f); if (empty($data)) { $res = false; logs(date('H:i:s', time()) . " the file: {$f} is empty."); } if (empty($data['MenDian'])) { $res = false; logs(date('H:i:s', time()) . " the file:' {$f} have not MenDian info."); } if (empty($data['MenDian']['ITEM'])) { $res = false; logs(date('H:i:s', time()) . " the file:' {$f} have not Item info."); } backFile($f); }