echo "AUTOCOMMIT SET TO TRUE\n";
$query = "select id, title, baseUrl, text, content from webpage where status = 2";
$results = $conn->query($query);
#bulk index using curl
$curlUrl = "http://" . $es_settings['host'] . ":" . $es_settings['port'] . "/" . $es_settings['index'] . "/webpage/_bulk";
echo "CURL URL: " . $curlUrl;
$ch = curl_init();
# initialize curl payload
$payload = "";
$rowNum = 1;
$rowCount = $results->num_rows;
# for all results of query
while ($row = mysqli_fetch_assoc($results)) {
echo "DOCUMENT NUMBER: " . $rowNum . "\n";
$metas = MetaParser::parseMetaTagsFromHtmlString($row['content'], ['description', 'keywords']);
$payloadHalf = json_encode(array("id" => utf8_encode($row['id']), "title" => utf8_encode($row['title']), "url" => utf8_encode($row['baseUrl']), "content" => utf8_encode($row['text']), "desc" => empty($metas['description']) ? null : utf8_encode($metas['description']), "keywords" => empty($metas['keywords']) ? null : utf8_encode($metas['keywords']), "subdomain" => MetaParser::getSubdomain($row['baseUrl'])), JSON_UNESCAPED_SLASHES);
$payload = $payload . "{\"create\":{}}\n" . $payloadHalf . "\n";
if ($rowNum % 200 == 0 || $rowNum == $rowCount) {
curl_setopt_array($ch, array(CURLOPT_CUSTOMREQUEST => 'POST', CURLOPT_URL => $curlUrl, CURLOPT_RETURNTRANSFER => 1, CURLOPT_POSTFIELDS => $payload));
echo "\nindexing job: starting...\n";
#execute bulk index for 200 documents
$response = curl_exec($ch);
echo $response;
echo "indexing job: done.\n";
$payload = "";
}
$rowNum++;
}
curl_close($ch);
# close the connection
$conn->close();
