/**
*
*/
public function spamlytics_new_daily_data()
{
$settings = \SpamLyticsHelper::get_settings();
if ($settings['api_key'] !== '') {
$fetcher = new Fetcher($settings['api_key']);
$fetcher->check_for_updates();
}
}
/**
* Sets the current data to be the skin of the player.
* @return self
*/
public function skin()
{
if ($this->data) {
$this->data = $this->data->reset();
return $this;
}
$contents = $this->fetcher->download($this->username);
$this->data = $this->provider->make($contents)->backup();
return $this;
}
public function search()
{
require_once 'src/Fetcher.php';
require_once 'src/Decoder.php';
$q = null;
$results = [];
if (isset($_GET['q'])) {
$q = $_GET['q'];
$fetcher = new Fetcher();
if ($text = $fetcher->search($q)) {
$decoder = new Decoder();
$results = $decoder->decode($text);
}
}
include_once 'html/index.html';
}
public function getMoviesFromDb($columnsNames = ["id", "name", "description", "rating"])
{
if (is_array($columnsNames) && !empty($columnsNames)) {
$columns = implode(",", $columnsNames);
$sqlQuery = "SELECT " . $columns . " FROM movies";
$conn = parent::getConnection();
$result = $conn->query($sqlQuery);
if ($result->num_rows > 0) {
while ($row = $result->fetch_assoc()) {
$movie = new Movie(parent::getConnection());
if (array_key_exists("id", $row)) {
$movie->setId($row["id"]);
}
if (array_key_exists("name", $row)) {
$movie->setName($row["name"]);
}
if (array_key_exists("description", $row)) {
$movie->setDescription($row["description"]);
}
if (array_key_exists("rating", $row)) {
$movie->setRating($row["rating"]);
}
$this->setArrayWithMovies($movie);
}
}
}
}
public static function cascadeCreate($names, $separator = " > ")
{
$array = explode($separator, $names);
foreach ($array as $index => $name) {
$array[$index] = trim($name);
}
$producttypearray = array();
//do first one (special treatment for root)
$producttype = Fetcher::fetchOne("Producttype", array('name' => '"' . $array[0] . '"'));
$parent_id = $producttype->getId();
$producttypearray[] = $producttype;
foreach ($array as $index => $name) {
//skip first one
if ($index == 0) {
continue;
}
//fetch next producttype based on previous
$producttype = Doctrine_Query::create()->from('Producttype p')->where('p.name="' . $name . '"')->andWhere('p.parent_id="' . $parent_id . '"')->fetchOne();
//one of the nodes not found
//if null, create
if (!$producttype) {
//create
$producttype = ProducttypeTable::createOne(array('name' => $name, 'parent_id' => $parent_id, 'priority' => 2));
}
//prepare for next round
$parent_id = $producttype->getId();
//add to array, whether fetched object or null
$producttypearray[] = $producttype;
}
return $producttypearray;
}
private function processline($line)
{
//save data from parsed inputstring to array
$productname = $line[1];
$object = Fetcher::fetchOne("Product", array('name' => '"' . $productname . '"'));
$producttype = $this->main->producttypedata->items[$line[3]]["object"];
$this->hydrate($productname, $line, $object, $producttype);
}
protected function set_fetched_data(&$signal, $element, $key = NULL)
{
if (is_array($element)) {
return parent::set_fetched_element($signal, $element, "array/" . $this->element_typemark(), $key);
} elseif (is_object($element)) {
return parent::set_fetched_element($signal, $element, "object/" . $this->element_typemark(), $key);
} else {
return false;
}
}
private function processLine($index, $line)
{
//save data from parsed inputstring to array
$name = $line[1];
//if this producttype is already in the array
if (array_key_exists($name, $this->items)) {
continue;
}
//search database for producttype by name
//$array=ProducttypeTable::followBreadCrumbsByNames($name," > ");
//$object=count($array)?$array[count($array)-1]:null;
$object = Fetcher::fetchOne("Producttype", array('path' => '"' . $name . '"'));
$this->items[$name] = array('name' => $name, 'description' => $line[2] ? $line[2] : ($object ? $object->getDescription() : ""), 'category1' => $line[3] ? $line[3] : ($object ? $object->getCategory1() : ""), 'category2' => $line[4] ? $line[4] : ($object ? $object->getCategory2() : ""), 'category3' => $line[5] ? $line[5] : ($object ? $object->getCategory3() : ""), 'category4' => $line[6] ? $line[6] : ($object ? $object->getCategory4() : ""), 'category5' => $line[7] ? $line[7] : ($object ? $object->getCategory5() : ""), 'category6' => $line[8] ? $line[8] : ($object ? $object->getCategory6() : ""), 'category7' => $line[9] ? $line[9] : ($object ? $object->getCategory7() : ""), 'category8' => $line[10] ? $line[10] : ($object ? $object->getCategory8() : ""), 'category9' => $line[11] ? $line[11] : ($object ? $object->getCategory9() : ""), 'category10' => $line[12] ? $line[12] : ($object ? $object->getCategory10() : ""), 'object' => $object, 'from' => "producttypesection");
}
/**
* Create a new CONTENTdm Fetcher Instance.
* @param array $settings configuration settings.
*/
public function __construct($settings)
{
parent::__construct($settings);
$this->key = $this->settings['record_key'];
$this->thumbnail = new \mik\filemanipulators\ThumbnailFromCdm($settings);
if (isset($settings['MANIPULATORS']['fetchermanipulators'])) {
$this->fetchermanipulators = $settings['MANIPULATORS']['fetchermanipulators'];
} else {
$this->fetchermanipulators = null;
}
if (!$this->createTempDirectory()) {
$this->log->addError("Cdm fetcher", array('Cannot create temp_directory' => $this->tempDirectory));
}
}
private function processAddProduct($index, $line)
{
//save data from parsed inputstring to array
$name = $line[1];
//search database for producttype by name
$producttype = Fetcher::fetchOne("Producttype", array('path' => '"' . $name . '"'));
//if not found
if (!$producttype) {
return;
}
$this->main->producttypedata->addProducttype($producttype);
//ask the product data group to add each product in this producttype
foreach ($producttype->getProducts() as $product) {
$this->main->productdata->addProduct($product, $producttype);
}
}
protected function process_data(&$signal)
{
parent::process_data($signal);
if ($this->core->browser->xfer($this->uri, $this->params, $this->files, $this->mustbeposted)) {
$data = $this->xferred_data($signal);
$type = $this->content_type($signal);
$result = $this->set_fetched_element($signal, $data, $type);
} else {
$result = !$this->stop_on_fail;
}
$extra_params = array('uri' => $uri, 'params' => $params, 'files' => $files, 'mustbeposted' => $mustbeposted);
if (isset($this->content_type)) {
$extra_params['content_type'] = $this->content_type;
}
$this->set_jobdata($result, $extra_params, $exclude_params);
return $result;
}
protected function process_data(&$signal)
{
if (!parent::process_data($signal)) {
return false;
}
$title = $signal->data_title($this->default_data_key);
if (is_null($title)) {
return false;
}
$filename = $this->core->info->title_filename($title);
$file = $this->read_file_from_wfstorage($filename, $this->wfstorages);
if ($file === false) {
return false;
}
$result = $this->set_fetched_element($signal, $file, "object/file");
$this->set_jobdata($result, array('filename' => $filename));
return $result;
}
public function getCinemasFromDb($columnsNames = ["id", "name", "address"])
{
if (is_array($columnsNames) && !empty($columnsNames)) {
$columns = implode(",", $columnsNames);
$sqlQuery = "SELECT " . $columns . " FROM cinemas";
$conn = parent::getConnection();
$result = $conn->query($sqlQuery);
if ($result->num_rows > 0) {
while ($row = $result->fetch_assoc()) {
$cinema = new Cinema(parent::getConnection());
if (array_key_exists("id", $row)) {
$cinema->setId($row["id"]);
}
if (array_key_exists("name", $row)) {
$cinema->setName($row["name"]);
}
if (array_key_exists("address", $row)) {
$cinema->setAddress($row["address"]);
}
$this->setArrayWithCinemas($cinema);
}
}
}
}
protected function build($urls)
{
$this->xml = new \DOMDocument('1.0', 'UTF-8');
$this->xml->preserveWhiteSpace = false;
$this->xml->formatOutput = true;
$urlsetNode = $this->xml->createElement('urlset');
$urlsetNode->setAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9');
$this->xml->appendChild($urlsetNode);
foreach ($urls as $url) {
$urlNode = $this->xml->createElement('url');
$locNode = $this->xml->createElement('loc');
$escapedUrl = $this->xml->createTextNode($this->protocol . ':' . Fetcher::formatUrl($url));
$locNode->appendChild($escapedUrl);
$urlNode->appendChild($locNode);
$lastmodNode = $this->xml->createElement('lastmod', date('Y-m-d'));
$urlNode->appendChild($lastmodNode);
$changefreqNode = $this->xml->createElement('changefreq', $this->changeFrequency);
$urlNode->appendChild($changefreqNode);
$priorityNode = $this->xml->createElement('priority', $this->getUrlPriority($url));
$urlNode->appendChild($priorityNode);
$this->xml->appendChild($urlNode);
}
return $this->xml;
}
public function getSimilarQuote()
{
return Fetcher::fetchOne("Quote", array('total' => $this->getUnittotal(), 'vendor_id' => SettingsTable::fetch("me_vendor_id"), 'product_id' => $this->getProductId()));
}
/**
* Поставнока подзагрузки в очередь, данные зависимы от главного загрузчика
*
* @param string $mapper
* @param string|null $src_key Если не указано, автогенерация из $mapper
* @param string|null $root_key Ключ родителя, из которого берется src_key
* @return ItemFetcher::create()
*/
public static function fetch($mapper, $src_key = null, $root_key = null)
{
return Fetcher::create($mapper, $src_key ? $src_key : strtolower($mapper) . '_id')->setRootKey($root_key);
}
<?php
require 'class.Fetcher.php';
$f = new Fetcher();
$f->load_proxylist(file_get_contents('proxy.txt'));
var_dump($f->curr_proxy);
var_dump($f->fetchs('http://home.peter23.com/ip2.php'));
exit(2);
}
try {
$actions = array();
foreach ($res->args['actions'] as $action) {
if ($action == 'crawl') {
$crawler = new Crawler();
$crawler->setShowLinksOnly($res->options['showLinksOnly']);
$actions[$action] = $crawler;
} else {
if ($action == 'index') {
$actions[$action] = new Indexer();
}
}
}
$fetcher = new Fetcher();
$retrieved = $fetcher->fetch($url, $actions, $res->options['force']);
if ($retrieved === false) {
exit(0);
}
$update = false;
foreach ($actions as $key => $action) {
Log::info("step: {$key}");
$update |= $action->run($retrieved);
}
if ($update) {
//FIXME: update index if it exists already
$fetcher->storeDoc($retrieved->url, $retrieved->esDoc);
} else {
Log::info("Not updating");
}
/**
* @return Berthe_AbstractVO[]
*/
public function getAll()
{
$pagi = new Fetcher(-1, -1);
$pagi->addSort('id', Fetcher::SORT_ASC);
return $this->getByPaginator($pagi)->getResultSet();
}
/**
* @param Fetcher $paginator
* @return array(string, array) the sql and the array of the parameters
*/
public function getSqlByPaginator(Fetcher $paginator)
{
list($filterInReq, $filterToParameter) = $paginator->getFiltersForQuery();
$sortInReq = $paginator->getSortForQuery();
$isRandom = $paginator->isRandomSort();
$limit = $paginator->getLimit();
if ($isRandom) {
$sql = <<<EOL
SELECT
id
FROM
(SELECT
id,
RANDOM()
FROM
{$this->getTableName()}
WHERE
{$filterInReq}
ORDER BY 2
{$limit}) randomized
EOL;
} else {
$sql = <<<EOL
SELECT
id
FROM
{$this->getTableName()}
WHERE
{$filterInReq}
ORDER BY
{$sortInReq}
{$limit}
EOL;
}
return array($sql, $filterToParameter);
}
/**
* @return VO[]
*/
public function getAll()
{
$fetcher = new Fetcher(-1, -1);
$fetcher->sortById(Fetcher::SORT_ASC);
return $this->getByFetcher($fetcher)->getResultSet();
}
public function testEchoPhrase()
{
$settings = array();
$a = new Fetcher($settings);
$this->assertEquals("It's fun to test code.", $a->echoPhrase("It's fun to test code."));
}
/**
* TODO optimize that one !
* @param Berthe_AbstractVO $vo
* @param Fetcher $paginator
* @param int $nbBefore
* @param int $nbAfter
* @param bool $loop
* @return array array[voBefore[], voAfter[]] BEFORE / AFTER
*/
public function getNextAndPreviousByPaginator(Berthe_AbstractVO $vo, Fetcher $paginator, $nbBefore = 1, $nbAfter = 1, $loop = false)
{
$page = $paginator->getPage();
$nbByPage = $paginator->getNbByPage();
$paginator->setPage(-1);
$paginator->setNbByPage(-1);
$ids = $this->_reader->selectByPaginator($paginator);
$paginator->setPage($page);
$paginator->setNbByPage($nbByPage);
$_keys = array_keys($ids, $vo->id);
$position = reset($_keys);
$beforeIds = $this->_fetchPrevious($position, $ids, $nbBefore, $loop);
$afterIds = $this->_fetchNext($position, $ids, $nbAfter, $loop);
return $this->_fetchNextAndPrevious($beforeIds, $afterIds);
}
public function processById($id)
{
$this->pricelist = Fetcher::fetchOne("Pricelist", array('id' => $id));
$this->hydrate();
//this needs to happen after the pricelist is loaded
$this->main->inputstring = $this->pricelist->getQuickInputString();
}
$summary[self::ENCODING] = $site[self::ENCODING];
$summary[self::HASH] = $link_id;
$summary[self::TYPE] = "link";
$summary[self::HTTP_CODE] = $link_keys;
$summary[self::LANG] = $lang;
$this->found_sites[self::LINK_SEEN_URLS][$part_num][] = $summary;
$link_word_lists = PhraseParser::extractPhrasesInLists($link_text, $lang);
$link_meta_ids = PhraseParser::calculateLinkMetas($url, $link_host, $link_text, $site_url);
if (!isset($this->found_sites[self::INVERTED_INDEX][$part_num])) {
$this->found_sites[self::INVERTED_INDEX][$part_num] = new IndexShard("fetcher_shard_{$part_num}");
}
$this->found_sites[self::INVERTED_INDEX][$part_num]->addDocumentWords($link_keys, self::NEEDS_OFFSET_FLAG, $link_word_lists, $link_meta_ids, PhraseParser::$materialized_metas, false, $link_rank);
}
}
$iterim_elapse = changeInMicrotime($interim_time);
if ($iterim_elapse > 5) {
crawlLog("..Inverting " . $site[self::URL] . "...took > 5s.");
}
crawlTimeoutLog("..Still building inverted index. Have processed " . "%s of %s documents.\nLast url processed was %s.", $i, $num_seen, $site[self::URL]);
}
if ($this->crawl_type == self::ARCHIVE_CRAWL) {
$this->recrawl_check_scheduler = true;
}
crawlLog(" Build mini inverted index time " . changeInMicrotime($start_time));
}
}
/*
* Instantiate and runs the Fetcher
*/
$fetcher = new Fetcher();
$fetcher->start();
public function getCinemaScreenings($id)
{
$sqlQuery = "SELECT id, cinema_id, movie_id, date FROM screenings WHERE date > NOW() AND cinema_id = {$id} ORDER BY date ";
$conn = parent::getConnection();
$result = $conn->query($sqlQuery);
if ($result->num_rows > 0) {
while (list($id, $cinemaId, $movieId, $date) = $result->fetch_array(MYSQLI_NUM)) {
$movie = new Movie($conn);
$movie->findMovieById($movieId);
$cinema = new Cinema($conn);
$cinema->findCinemaById($cinemaId);
$screening = new Screening($conn);
$screening->setMovie($movie);
$screening->setCinema($cinema);
$screening->setDate($date);
$screening->setId($id);
$this->setArrayWithCinemaScreenings($screening);
}
}
}
Fetcher::$spanHosts = isset($opts['span-hosts']) || isset($opts['H']);
Fetcher::$quiet = isset($opts['quiet']) || isset($opts['q']);
Fetcher::$recursive = isset($opts['recursive']) || isset($opts['r']);
Fetcher::$verbose = isset($opts['verbose']) || isset($opts['v']);
if (isset($opts['connect-timeout'])) {
Fetcher::$connectTimeout = $opts['connect-timeout'];
}
if (isset($opts['max-redirect'])) {
Fetcher::$maxRedirect = $opts['max-redirect'];
}
if (isset($opts['level']) || isset($opts['l'])) {
Fetcher::$maxDepth = isset($opts['level']) ? $opts['level'] : $opts['l'];
}
if (isset($opts['output-directory']) || isset($opts['o'])) {
Fetcher::$outputDirectory = isset($opts['output-directory']) ? $opts['output-directory'] : $opts['o'];
}
if (isset($opts['timeout']) || isset($opts['T'])) {
Fetcher::$timeout = isset($opts['timeout']) ? $opts['timeout'] : $opts['T'];
}
if (isset($opts['wait']) || isset($opts['w'])) {
Fetcher::$wait = isset($opts['wait']) ? $opts['wait'] : $opts['w'];
}
try {
Fetcher::fetch($url);
} catch (Exception $e) {
if (!Fetcher::$quiet) {
fwrite(STDERR, '[error] ' . $e->getMessage() . "\n");
}
exit(1);
}
exit(0);
