/**
* updates a given source
* returns an error or true on success
*
* @return void
* @param mixed $source the current source
*/
public function fetch($source)
{
@set_time_limit(5000);
@error_reporting(E_ERROR);
// logging
\F3::get('logger')->log('---', \DEBUG);
\F3::get('logger')->log('start fetching source "' . $source['title'] . ' (id: ' . $source['id'] . ') ', \DEBUG);
// get spout
$spoutLoader = new \helpers\SpoutLoader();
$spout = $spoutLoader->get($source['spout']);
if ($spout === false) {
\F3::get('logger')->log('unknown spout: ' . $source['spout'], \ERROR);
return;
}
\F3::get('logger')->log('spout successfully loaded: ' . $source['spout'], \DEBUG);
// receive content
\F3::get('logger')->log('fetch content', \DEBUG);
try {
$spout->load(json_decode(html_entity_decode($source['params']), true));
} catch (\exception $e) {
\F3::get('logger')->log('error loading feed content for ' . $source['title'] . ': ' . $e->getMessage(), \ERROR);
$this->sourceDao->error($source['id'], date('Y-m-d H:i:s') . 'error loading feed content: ' . $e->getMessage());
return;
}
// current date
$minDate = new \DateTime();
$minDate->sub(new \DateInterval('P' . \F3::get('items_lifetime') . 'D'));
\F3::get('logger')->log('minimum date: ' . $minDate->format('Y-m-d H:i:s'), \DEBUG);
// insert new items in database
\F3::get('logger')->log('start item fetching', \DEBUG);
$itemsInFeed = array();
foreach ($spout as $item) {
$itemsInFeed[] = $item->getId();
}
$itemsFound = $this->itemsDao->findAll($itemsInFeed);
$lasticon = false;
foreach ($spout as $item) {
// item already in database?
if (isset($itemsFound[$item->getId()])) {
continue;
}
// test date: continue with next if item too old
$itemDate = new \DateTime($item->getDate());
if ($itemDate < $minDate) {
\F3::get('logger')->log('item "' . $item->getTitle() . '" (' . $item->getDate() . ') older than ' . \F3::get('items_lifetime') . ' days', \DEBUG);
continue;
}
// date in future? Set current date
$now = new \DateTime();
if ($itemDate > $now) {
$itemDate = $now;
}
// insert new item
\F3::get('logger')->log('start insertion of new item "' . $item->getTitle() . '"', \DEBUG);
$content = "";
try {
// fetch content
$content = $item->getContent();
// sanitize content html
$content = $this->sanitizeContent($content);
} catch (\exception $e) {
$content = 'Error: Content not fetched. Reason: ' . $e->getMessage();
\F3::get('logger')->log('Can not fetch "' . $item->getTitle() . '" : ' . $e->getMessage(), \ERROR);
}
// sanitize title
$title = $this->sanitizeContent($item->getTitle());
if (strlen(trim($title)) == 0) {
$title = "[" . \F3::get('lang_no_title') . "]";
}
// sanitize author
$author = htmlspecialchars_decode($item->getAuthor());
$author = htmLawed($author, array("deny_attribute" => "*", "elements" => "-*"));
\F3::get('logger')->log('item content sanitized', \DEBUG);
try {
$icon = $item->getIcon();
} catch (\exception $e) {
return;
}
$newItem = array('title' => $title, 'content' => $content, 'source' => $source['id'], 'datetime' => $itemDate->format('Y-m-d H:i:s'), 'uid' => $item->getId(), 'thumbnail' => $item->getThumbnail(), 'icon' => $icon !== false ? $icon : "", 'link' => htmLawed($item->getLink(), array("deny_attribute" => "*", "elements" => "-*")), 'author' => $author);
// save thumbnail
$newItem = $this->fetchThumbnail($item->getThumbnail(), $newItem);
// save icon
$newItem = $this->fetchIcon($item->getIcon(), $newItem, $lasticon);
// insert new item
$this->itemsDao->add($newItem);
\F3::get('logger')->log('item inserted', \DEBUG);
\F3::get('logger')->log('Memory usage: ' . memory_get_usage(), \DEBUG);
\F3::get('logger')->log('Memory peak usage: ' . memory_get_peak_usage(), \DEBUG);
}
// destroy feed object (prevent memory issues)
\F3::get('logger')->log('destroy spout object', \DEBUG);
$spout->destroy();
// remove previous errors and set last update timestamp
$this->updateSource($source);
}
/**
* validate new data for a given source
*
* @return bool|mixed true on succes or array of
* errors on failure
* @param string $title
* @param string $spout
* @param mixed $params
*
* @author Tobias Zeising
*/
public function validate($title, $spout, $params)
{
$result = array();
// title
if (strlen(trim($title)) == 0) {
$result['title'] = 'no text for title given';
}
// spout type
$spoutLoader = new \helpers\SpoutLoader();
$spout = $spoutLoader->get($spout);
if ($spout == false) {
$result['spout'] = 'invalid spout type';
// check params
} else {
// params given but not expected
if ($spout->params === false) {
if (is_array($spout->params) && count($spout->params) > 0) {
$result['spout'] = 'this spout doesn\'t expect any param';
}
}
if ($spout->params == false) {
if (count($result) > 0) {
return $result;
}
return true;
}
// required but not given params
foreach ($spout->params as $id => $param) {
if ($param['required'] === false) {
continue;
}
$found = false;
foreach ($params as $userParamId => $userParamValue) {
if ($userParamId == $id) {
$found = true;
}
}
if ($found == false) {
$result[$id] = 'param ' . $param['title'] . ' required but not given';
}
}
// given params valid?
foreach ($params as $id => $value) {
$validation = $spout->params[$id]['validation'];
if (!is_array($validation)) {
$validation = array($validation);
}
foreach ($validation as $validate) {
if ($validate == 'alpha' && !preg_match("[A-Za-Z._\\b]+", $value)) {
$result[$id] = 'only alphabetic characters allowed for ' . $spout->params[$id]['title'];
} else {
if ($validate == 'email' && !preg_match('/^[^0-9][a-zA-Z0-9_]+([.][a-zA-Z0-9_]+)*[@][a-zA-Z0-9_]+([.][a-zA-Z0-9_]+)*[.][a-zA-Z]{2,4}$/', $value)) {
$result[$id] = $spout->params[$id]['title'] . ' is not a valid email address';
} else {
if ($validate == 'numeric' && !is_numeric($value)) {
$result[$id] = 'only numeric values allowed for ' . $spout->params[$id]['title'];
} else {
if ($validate == 'int' && intval($value) != $value) {
$result[$id] = 'only integer values allowed for ' . $spout->params[$id]['title'];
} else {
if ($validate == 'alnum' && !preg_match("[A-Za-Z0-9._\\b]+", $value)) {
$result[$id] = 'only alphanumeric values allowed for ' . $spout->params[$id]['title'];
} else {
if ($validate == 'notempty' && strlen(trim($value)) == 0) {
$result[$id] = 'empty value for ' . $spout->params[$id]['title'] . ' not allowed';
}
}
}
}
}
}
}
}
// select: user sent value which is not a predefined option?
foreach ($params as $id => $value) {
if ($spout->params[$id]['type'] != "select") {
continue;
}
$values = $spout->params[$id]['values'];
$found = false;
foreach ($values as $optionName => $optionTitle) {
if ($optionName == $value) {
$found = true;
}
}
if ($found == false) {
$result[$id] = 'param ' . $spout->params[$id]['title'] . ' was not set to a predefined value';
}
}
}
if (count($result) > 0) {
return $result;
}
return true;
}
/**
* returns all sources including last icon
*
* @return mixed all sources
*/
public function getWithIcon()
{
$ret = \F3::get('db')->exec('SELECT
sources.id, sources.title, sources.tags, sources.spout,
sources.params, sources.filter, sources.error, sources.lastentry,
sourceicons.icon AS icon
FROM ' . \F3::get('db_prefix') . 'sources AS sources
LEFT OUTER JOIN
(SELECT items.source, icon
FROM ' . \F3::get('db_prefix') . 'items AS items,
(SELECT source, MAX(id) as maxid
FROM ' . \F3::get('db_prefix') . 'items AS items
WHERE icon IS NOT NULL AND icon != \'\'
GROUP BY items.source) AS icons
WHERE items.id=icons.maxid AND items.source=icons.source
) AS sourceicons
ON sources.id=sourceicons.source
ORDER BY ' . $this->stmt->nullFirst('sources.error', 'DESC') . ', lower(sources.title)');
$spoutLoader = new \helpers\SpoutLoader();
for ($i = 0; $i < count($ret); $i++) {
$ret[$i]['spout_obj'] = $spoutLoader->get($ret[$i]['spout']);
}
return $ret;
}
/**
* returns all available spouts
* json
*
* @return void
*/
public function spouts()
{
$this->needsLoggedIn();
$spoutLoader = new \helpers\SpoutLoader();
$spouts = $spoutLoader->all();
$this->view->jsonSuccess($spouts);
}
/**
* returns all sources
*
* @return mixed all sources
*/
public function get()
{
$ret = \F3::get('db')->exec('SELECT id, title, tags, spout, params, error FROM ' . \F3::get('db_prefix') . 'sources ORDER BY error DESC, lower(title) ASC');
$spoutLoader = new \helpers\SpoutLoader();
for ($i = 0; $i < count($ret); $i++) {
$ret[$i]['spout_obj'] = $spoutLoader->get($ret[$i]['spout']);
}
return $ret;
}
