/** * */ public function spamlytics_new_daily_data() { $settings = \SpamLyticsHelper::get_settings(); if ($settings['api_key'] !== '') { $fetcher = new Fetcher($settings['api_key']); $fetcher->check_for_updates(); } }
/** * Sets the current data to be the skin of the player. * @return self */ public function skin() { if ($this->data) { $this->data = $this->data->reset(); return $this; } $contents = $this->fetcher->download($this->username); $this->data = $this->provider->make($contents)->backup(); return $this; }
public function search() { require_once 'src/Fetcher.php'; require_once 'src/Decoder.php'; $q = null; $results = []; if (isset($_GET['q'])) { $q = $_GET['q']; $fetcher = new Fetcher(); if ($text = $fetcher->search($q)) { $decoder = new Decoder(); $results = $decoder->decode($text); } } include_once 'html/index.html'; }
public function getMoviesFromDb($columnsNames = ["id", "name", "description", "rating"]) { if (is_array($columnsNames) && !empty($columnsNames)) { $columns = implode(",", $columnsNames); $sqlQuery = "SELECT " . $columns . " FROM movies"; $conn = parent::getConnection(); $result = $conn->query($sqlQuery); if ($result->num_rows > 0) { while ($row = $result->fetch_assoc()) { $movie = new Movie(parent::getConnection()); if (array_key_exists("id", $row)) { $movie->setId($row["id"]); } if (array_key_exists("name", $row)) { $movie->setName($row["name"]); } if (array_key_exists("description", $row)) { $movie->setDescription($row["description"]); } if (array_key_exists("rating", $row)) { $movie->setRating($row["rating"]); } $this->setArrayWithMovies($movie); } } } }
public static function cascadeCreate($names, $separator = " > ") { $array = explode($separator, $names); foreach ($array as $index => $name) { $array[$index] = trim($name); } $producttypearray = array(); //do first one (special treatment for root) $producttype = Fetcher::fetchOne("Producttype", array('name' => '"' . $array[0] . '"')); $parent_id = $producttype->getId(); $producttypearray[] = $producttype; foreach ($array as $index => $name) { //skip first one if ($index == 0) { continue; } //fetch next producttype based on previous $producttype = Doctrine_Query::create()->from('Producttype p')->where('p.name="' . $name . '"')->andWhere('p.parent_id="' . $parent_id . '"')->fetchOne(); //one of the nodes not found //if null, create if (!$producttype) { //create $producttype = ProducttypeTable::createOne(array('name' => $name, 'parent_id' => $parent_id, 'priority' => 2)); } //prepare for next round $parent_id = $producttype->getId(); //add to array, whether fetched object or null $producttypearray[] = $producttype; } return $producttypearray; }
private function processline($line) { //save data from parsed inputstring to array $productname = $line[1]; $object = Fetcher::fetchOne("Product", array('name' => '"' . $productname . '"')); $producttype = $this->main->producttypedata->items[$line[3]]["object"]; $this->hydrate($productname, $line, $object, $producttype); }
protected function set_fetched_data(&$signal, $element, $key = NULL) { if (is_array($element)) { return parent::set_fetched_element($signal, $element, "array/" . $this->element_typemark(), $key); } elseif (is_object($element)) { return parent::set_fetched_element($signal, $element, "object/" . $this->element_typemark(), $key); } else { return false; } }
private function processLine($index, $line) { //save data from parsed inputstring to array $name = $line[1]; //if this producttype is already in the array if (array_key_exists($name, $this->items)) { continue; } //search database for producttype by name //$array=ProducttypeTable::followBreadCrumbsByNames($name," > "); //$object=count($array)?$array[count($array)-1]:null; $object = Fetcher::fetchOne("Producttype", array('path' => '"' . $name . '"')); $this->items[$name] = array('name' => $name, 'description' => $line[2] ? $line[2] : ($object ? $object->getDescription() : ""), 'category1' => $line[3] ? $line[3] : ($object ? $object->getCategory1() : ""), 'category2' => $line[4] ? $line[4] : ($object ? $object->getCategory2() : ""), 'category3' => $line[5] ? $line[5] : ($object ? $object->getCategory3() : ""), 'category4' => $line[6] ? $line[6] : ($object ? $object->getCategory4() : ""), 'category5' => $line[7] ? $line[7] : ($object ? $object->getCategory5() : ""), 'category6' => $line[8] ? $line[8] : ($object ? $object->getCategory6() : ""), 'category7' => $line[9] ? $line[9] : ($object ? $object->getCategory7() : ""), 'category8' => $line[10] ? $line[10] : ($object ? $object->getCategory8() : ""), 'category9' => $line[11] ? $line[11] : ($object ? $object->getCategory9() : ""), 'category10' => $line[12] ? $line[12] : ($object ? $object->getCategory10() : ""), 'object' => $object, 'from' => "producttypesection"); }
/** * Create a new CONTENTdm Fetcher Instance. * @param array $settings configuration settings. */ public function __construct($settings) { parent::__construct($settings); $this->key = $this->settings['record_key']; $this->thumbnail = new \mik\filemanipulators\ThumbnailFromCdm($settings); if (isset($settings['MANIPULATORS']['fetchermanipulators'])) { $this->fetchermanipulators = $settings['MANIPULATORS']['fetchermanipulators']; } else { $this->fetchermanipulators = null; } if (!$this->createTempDirectory()) { $this->log->addError("Cdm fetcher", array('Cannot create temp_directory' => $this->tempDirectory)); } }
private function processAddProduct($index, $line) { //save data from parsed inputstring to array $name = $line[1]; //search database for producttype by name $producttype = Fetcher::fetchOne("Producttype", array('path' => '"' . $name . '"')); //if not found if (!$producttype) { return; } $this->main->producttypedata->addProducttype($producttype); //ask the product data group to add each product in this producttype foreach ($producttype->getProducts() as $product) { $this->main->productdata->addProduct($product, $producttype); } }
protected function process_data(&$signal) { parent::process_data($signal); if ($this->core->browser->xfer($this->uri, $this->params, $this->files, $this->mustbeposted)) { $data = $this->xferred_data($signal); $type = $this->content_type($signal); $result = $this->set_fetched_element($signal, $data, $type); } else { $result = !$this->stop_on_fail; } $extra_params = array('uri' => $uri, 'params' => $params, 'files' => $files, 'mustbeposted' => $mustbeposted); if (isset($this->content_type)) { $extra_params['content_type'] = $this->content_type; } $this->set_jobdata($result, $extra_params, $exclude_params); return $result; }
protected function process_data(&$signal) { if (!parent::process_data($signal)) { return false; } $title = $signal->data_title($this->default_data_key); if (is_null($title)) { return false; } $filename = $this->core->info->title_filename($title); $file = $this->read_file_from_wfstorage($filename, $this->wfstorages); if ($file === false) { return false; } $result = $this->set_fetched_element($signal, $file, "object/file"); $this->set_jobdata($result, array('filename' => $filename)); return $result; }
public function getCinemasFromDb($columnsNames = ["id", "name", "address"]) { if (is_array($columnsNames) && !empty($columnsNames)) { $columns = implode(",", $columnsNames); $sqlQuery = "SELECT " . $columns . " FROM cinemas"; $conn = parent::getConnection(); $result = $conn->query($sqlQuery); if ($result->num_rows > 0) { while ($row = $result->fetch_assoc()) { $cinema = new Cinema(parent::getConnection()); if (array_key_exists("id", $row)) { $cinema->setId($row["id"]); } if (array_key_exists("name", $row)) { $cinema->setName($row["name"]); } if (array_key_exists("address", $row)) { $cinema->setAddress($row["address"]); } $this->setArrayWithCinemas($cinema); } } } }
protected function build($urls) { $this->xml = new \DOMDocument('1.0', 'UTF-8'); $this->xml->preserveWhiteSpace = false; $this->xml->formatOutput = true; $urlsetNode = $this->xml->createElement('urlset'); $urlsetNode->setAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9'); $this->xml->appendChild($urlsetNode); foreach ($urls as $url) { $urlNode = $this->xml->createElement('url'); $locNode = $this->xml->createElement('loc'); $escapedUrl = $this->xml->createTextNode($this->protocol . ':' . Fetcher::formatUrl($url)); $locNode->appendChild($escapedUrl); $urlNode->appendChild($locNode); $lastmodNode = $this->xml->createElement('lastmod', date('Y-m-d')); $urlNode->appendChild($lastmodNode); $changefreqNode = $this->xml->createElement('changefreq', $this->changeFrequency); $urlNode->appendChild($changefreqNode); $priorityNode = $this->xml->createElement('priority', $this->getUrlPriority($url)); $urlNode->appendChild($priorityNode); $this->xml->appendChild($urlNode); } return $this->xml; }
public function getSimilarQuote() { return Fetcher::fetchOne("Quote", array('total' => $this->getUnittotal(), 'vendor_id' => SettingsTable::fetch("me_vendor_id"), 'product_id' => $this->getProductId())); }
/** * Поставнока подзагрузки в очередь, данные зависимы от главного загрузчика * * @param string $mapper * @param string|null $src_key Если не указано, автогенерация из $mapper * @param string|null $root_key Ключ родителя, из которого берется src_key * @return ItemFetcher::create() */ public static function fetch($mapper, $src_key = null, $root_key = null) { return Fetcher::create($mapper, $src_key ? $src_key : strtolower($mapper) . '_id')->setRootKey($root_key); }
<?php require 'class.Fetcher.php'; $f = new Fetcher(); $f->load_proxylist(file_get_contents('proxy.txt')); var_dump($f->curr_proxy); var_dump($f->fetchs('http://home.peter23.com/ip2.php'));
exit(2); } try { $actions = array(); foreach ($res->args['actions'] as $action) { if ($action == 'crawl') { $crawler = new Crawler(); $crawler->setShowLinksOnly($res->options['showLinksOnly']); $actions[$action] = $crawler; } else { if ($action == 'index') { $actions[$action] = new Indexer(); } } } $fetcher = new Fetcher(); $retrieved = $fetcher->fetch($url, $actions, $res->options['force']); if ($retrieved === false) { exit(0); } $update = false; foreach ($actions as $key => $action) { Log::info("step: {$key}"); $update |= $action->run($retrieved); } if ($update) { //FIXME: update index if it exists already $fetcher->storeDoc($retrieved->url, $retrieved->esDoc); } else { Log::info("Not updating"); }
/** * @return Berthe_AbstractVO[] */ public function getAll() { $pagi = new Fetcher(-1, -1); $pagi->addSort('id', Fetcher::SORT_ASC); return $this->getByPaginator($pagi)->getResultSet(); }
/** * @param Fetcher $paginator * @return array(string, array) the sql and the array of the parameters */ public function getSqlByPaginator(Fetcher $paginator) { list($filterInReq, $filterToParameter) = $paginator->getFiltersForQuery(); $sortInReq = $paginator->getSortForQuery(); $isRandom = $paginator->isRandomSort(); $limit = $paginator->getLimit(); if ($isRandom) { $sql = <<<EOL SELECT id FROM (SELECT id, RANDOM() FROM {$this->getTableName()} WHERE {$filterInReq} ORDER BY 2 {$limit}) randomized EOL; } else { $sql = <<<EOL SELECT id FROM {$this->getTableName()} WHERE {$filterInReq} ORDER BY {$sortInReq} {$limit} EOL; } return array($sql, $filterToParameter); }
/** * @return VO[] */ public function getAll() { $fetcher = new Fetcher(-1, -1); $fetcher->sortById(Fetcher::SORT_ASC); return $this->getByFetcher($fetcher)->getResultSet(); }
public function testEchoPhrase() { $settings = array(); $a = new Fetcher($settings); $this->assertEquals("It's fun to test code.", $a->echoPhrase("It's fun to test code.")); }
/** * TODO optimize that one ! * @param Berthe_AbstractVO $vo * @param Fetcher $paginator * @param int $nbBefore * @param int $nbAfter * @param bool $loop * @return array array[voBefore[], voAfter[]] BEFORE / AFTER */ public function getNextAndPreviousByPaginator(Berthe_AbstractVO $vo, Fetcher $paginator, $nbBefore = 1, $nbAfter = 1, $loop = false) { $page = $paginator->getPage(); $nbByPage = $paginator->getNbByPage(); $paginator->setPage(-1); $paginator->setNbByPage(-1); $ids = $this->_reader->selectByPaginator($paginator); $paginator->setPage($page); $paginator->setNbByPage($nbByPage); $_keys = array_keys($ids, $vo->id); $position = reset($_keys); $beforeIds = $this->_fetchPrevious($position, $ids, $nbBefore, $loop); $afterIds = $this->_fetchNext($position, $ids, $nbAfter, $loop); return $this->_fetchNextAndPrevious($beforeIds, $afterIds); }
public function processById($id) { $this->pricelist = Fetcher::fetchOne("Pricelist", array('id' => $id)); $this->hydrate(); //this needs to happen after the pricelist is loaded $this->main->inputstring = $this->pricelist->getQuickInputString(); }
$summary[self::ENCODING] = $site[self::ENCODING]; $summary[self::HASH] = $link_id; $summary[self::TYPE] = "link"; $summary[self::HTTP_CODE] = $link_keys; $summary[self::LANG] = $lang; $this->found_sites[self::LINK_SEEN_URLS][$part_num][] = $summary; $link_word_lists = PhraseParser::extractPhrasesInLists($link_text, $lang); $link_meta_ids = PhraseParser::calculateLinkMetas($url, $link_host, $link_text, $site_url); if (!isset($this->found_sites[self::INVERTED_INDEX][$part_num])) { $this->found_sites[self::INVERTED_INDEX][$part_num] = new IndexShard("fetcher_shard_{$part_num}"); } $this->found_sites[self::INVERTED_INDEX][$part_num]->addDocumentWords($link_keys, self::NEEDS_OFFSET_FLAG, $link_word_lists, $link_meta_ids, PhraseParser::$materialized_metas, false, $link_rank); } } $iterim_elapse = changeInMicrotime($interim_time); if ($iterim_elapse > 5) { crawlLog("..Inverting " . $site[self::URL] . "...took > 5s."); } crawlTimeoutLog("..Still building inverted index. Have processed " . "%s of %s documents.\nLast url processed was %s.", $i, $num_seen, $site[self::URL]); } if ($this->crawl_type == self::ARCHIVE_CRAWL) { $this->recrawl_check_scheduler = true; } crawlLog(" Build mini inverted index time " . changeInMicrotime($start_time)); } } /* * Instantiate and runs the Fetcher */ $fetcher = new Fetcher(); $fetcher->start();
public function getCinemaScreenings($id) { $sqlQuery = "SELECT id, cinema_id, movie_id, date FROM screenings WHERE date > NOW() AND cinema_id = {$id} ORDER BY date "; $conn = parent::getConnection(); $result = $conn->query($sqlQuery); if ($result->num_rows > 0) { while (list($id, $cinemaId, $movieId, $date) = $result->fetch_array(MYSQLI_NUM)) { $movie = new Movie($conn); $movie->findMovieById($movieId); $cinema = new Cinema($conn); $cinema->findCinemaById($cinemaId); $screening = new Screening($conn); $screening->setMovie($movie); $screening->setCinema($cinema); $screening->setDate($date); $screening->setId($id); $this->setArrayWithCinemaScreenings($screening); } } }
Fetcher::$spanHosts = isset($opts['span-hosts']) || isset($opts['H']); Fetcher::$quiet = isset($opts['quiet']) || isset($opts['q']); Fetcher::$recursive = isset($opts['recursive']) || isset($opts['r']); Fetcher::$verbose = isset($opts['verbose']) || isset($opts['v']); if (isset($opts['connect-timeout'])) { Fetcher::$connectTimeout = $opts['connect-timeout']; } if (isset($opts['max-redirect'])) { Fetcher::$maxRedirect = $opts['max-redirect']; } if (isset($opts['level']) || isset($opts['l'])) { Fetcher::$maxDepth = isset($opts['level']) ? $opts['level'] : $opts['l']; } if (isset($opts['output-directory']) || isset($opts['o'])) { Fetcher::$outputDirectory = isset($opts['output-directory']) ? $opts['output-directory'] : $opts['o']; } if (isset($opts['timeout']) || isset($opts['T'])) { Fetcher::$timeout = isset($opts['timeout']) ? $opts['timeout'] : $opts['T']; } if (isset($opts['wait']) || isset($opts['w'])) { Fetcher::$wait = isset($opts['wait']) ? $opts['wait'] : $opts['w']; } try { Fetcher::fetch($url); } catch (Exception $e) { if (!Fetcher::$quiet) { fwrite(STDERR, '[error] ' . $e->getMessage() . "\n"); } exit(1); } exit(0);