-
Notifications
You must be signed in to change notification settings - Fork 0
/
crawl.php
executable file
·64 lines (46 loc) · 1.1 KB
/
crawl.php
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
<?php
include 'libs/general.php';
CronManager::init(__FILE__, 10);
try
{
// GET URLs
$urls = Providers::get_lead_urls();
_w('Got ' . count($urls) . ' urls');
/// PROCESS URLS
foreach ($urls as $url) {
// Parse array to url
$url_w = urls::create_url($url);
_w('Getting url ' . $url_w);
// Init content analyzer
$ca = ContentAnalyzer::getAnalyzer($url_w);
// If content ignored
if (!$ca) {
_w('ignored');
// Skip
continue;
}
// _w($ca->getOgDescription());
// die;
_w('Createing general CA data');
_w('populating search table');
if (Providers::create_search_item($ca)) {
_w("search data inserted");
} else {
_w("unable to insert search data");
}
_w('setting status to indexed');
if (Providers::change_url_status($url_w, Providers::URLS_TYPE_INDEXED)) {
_w('status changed to INDEXED');
} else {
_w('unable to change status');
}
// _w($ca->getLinks());
// die;
_w('inserting all other urls');
Providers::insert_url_list($ca->getLinks());
}
} catch (Exception $ex) {
_w('WAS ERROR !!! ' . $ex->getMessage());
}
_w('Done for now');
unset($urls);