function __construct($url, $feedId, $enableCacheOnly = false)
{
$this->cachedir = Config::get('cachedir');
if (!is_dir($this->cachedir)) {
throw new Exception('Cache dir not present');
}
if (!is_writable($this->cachedir)) {
throw new Exception('Cache dir not writable');
}
$cachefile = $this->cachedir . $feedId;
// echo 'Cache dir: ' . $cachefile;
// exit;
try {
if (!$enableCacheOnly) {
DiscoUtils::debug('Downloading metadata from ' . tc_colored($url, 'green') . " and storing cache at " . tc_colored($cachefile, 'green'));
$data = @file_get_contents($url);
if ($data === false) {
throw new Exception('Error retrieving metadata from ' . $url);
}
file_put_contents($cachefile, $data);
} else {
DiscoUtils::debug('Looking up cached metadata from ' . tc_colored($cachefile, 'green'));
}
} catch (Exception $e) {
error_log('Error updating metadata from source ' . $feedId . ' : ' . $e->getMessage());
}
if (!file_exists($cachefile)) {
throw new Exception('Not able to continue processing this feed, because cannot read cached file');
}
DiscoUtils::debug('Metadata ready, starting to parse XML and validate document');
$this->list = array();
$entities = SimpleSAML_Metadata_SAMLParser::parseDescriptorsFile($cachefile);
foreach ($entities as $entityId => $entity) {
$md = $entity->getMetadata1xIdP();
if ($md !== NULL) {
$this->list[$entityId] = $md;
}
$md = $entity->getMetadata20IdP();
if ($md !== NULL) {
$this->list[$entityId] = $md;
}
$this->processSPEntity($entity->getMetadata20SP());
}
if (count($this->list) === 0) {
throw new Exception('No entities found at URL ' . $src);
}
}
public function process()
{
$start = microtime(true);
$this->idplist = array();
$processedEntities = array();
$i = 0;
foreach ($this->metadata as $entityId => $metadataEntry) {
// if (rand(0, 2) === 2) {
// DiscoUtils::log('Skipping random entity: ' . $entityId); continue;
// }
DiscoUtils::log('Processing ' . $entityId);
$entry = new FeedItem($entityId, $this->feedId, $metadataEntry, $this->feedconfig);
$entry->process();
$data = $entry->getJSON();
// echo json_encode($data['disco'], JSON_PRETTY_PRINT) . "\n\n";
$this->idplist[] = $entry;
$processedEntities[] = $entityId;
// if ($i++ > 5) break;
}
foreach ($this->idplist as $item) {
$this->store->insertOrUpdate($item);
}
$existingEntries = $this->store->listFeedEntities($this->feedId);
$toDelete = array_diff($existingEntries, $processedEntities);
foreach ($toDelete as $td) {
DiscoUtils::log('Removing entityId ' . $td);
$this->store->remove($this->feedId, $td);
}
if (count($toDelete) === count($processedEntities)) {
throw new Exception('Will not delete all entities. We assume there is a mistake somewhere... Please fix ASAP.');
}
if (count($toDelete) === 0) {
DiscoUtils::log('All existing entities found, not removing any existing from ' . $this->feedId);
}
$end = microtime(true);
$timer = $end - $start;
$this->store->logProcess($this->feedId, $timer);
}
$pipe = Pipe::fromDB($c);
$query = $pipe->getQuery();
$response = $store->getIdPs($query);
// $response = $query;
// $response = $store->getFeed($parameters[1]);
} else {
if (DiscoUtils::route('get', '^/feed/([a-z0-9\\-_]+)/disco$', $parameters, $body)) {
$response = $store->getFeed($parameters[1]);
} else {
if (DiscoUtils::route('get', '^/feed/([a-z0-9\\-_]+)/metadata$', $parameters, $body)) {
$response = $store->getFeedMetadata($parameters[1]);
} else {
if (DiscoUtils::route('get', '^/apps$', $parameters, $body)) {
$response = array('foo' => 'bar');
} else {
if (DiscoUtils::route('get', '^/logo$', $parameters, $qs)) {
if (!isset($_REQUEST['entityId'])) {
throw new Exception('Missing required parameter entityId');
}
if (!isset($_REQUEST['feed'])) {
throw new Exception('Missing required parameter feed');
}
$data = $logostore->get($_REQUEST['entityId'], $_REQUEST['feed'], true);
// $data = $logostore->get('https://pieter.aai.surfnet.nl/simplesamlphp/saml2/idp/metadata.php', 'surfnet2', true);
header('Content-Type: image/png');
echo $data['logo']->bin;
exit;
} else {
throw new Exception('Invalid request');
}
}
public static function logConsole($en = true)
{
self::$logConsole = $en;
}
#!/usr/bin/env php
<?php
require dirname(dirname(__FILE__)) . '/lib/autoload.php';
$command = new Commando\Command();
$command->option()->describedAs('Command to run: default is update.');
$command->option('f')->aka('feed')->describedAs('The feed identifier to load.');
$command->option('cache-only')->boolean()->describedAs('Do not load metadata, only use existing cache.');
if ($command[0] === 'termcolor') {
phpterm_demo();
exit;
}
$backend = new DiscoJuiceBackend();
if ($command['cache-only']) {
DiscoUtils::log("Running in cache-only mode");
$backend->enableCacheOnly(true);
}
DiscoUtils::log("DiscoJuice update script. Now updating metadata.", true);
if ($command['feed']) {
$backend->updateFeed($command['feed']);
} else {
$backend->update();
}
} else {
if (DiscoUtils::route('get', '^/reviews/([^/]+)$', $parameters, $qs)) {
} else {
if (DiscoUtils::route('post', '^/reviews/([^/]+)$', $parameters, $qs)) {
} else {
if (DiscoUtils::route('get', '^/favs$', $parameters, $qs)) {
$userid = $gk->getUserID();
$fav = Favourites::getByID($userid);
$data = $fav->getView();
if (!empty($data) && isset($data['favs'])) {
$response = $data['favs'];
} else {
$response = array();
}
} else {
if (DiscoUtils::route('post', '^/favs$', $parameters, $qs)) {
// $inputraw = file_get_contents("php://input");
$userid = $gk->getUserID();
// $response = array(
// 'userid' => $userid,
// 'data' => $inputraw,
// 'qs' => $qs,
// 'parameters' => $parameters,
// 'headers' => getallheaders(),
// );
$fav = new Favourites(array('id' => $userid, 'favs' => $qs));
$fav->save();
$fav = Favourites::getByID($userid);
$data = $fav->getView();
if (!empty($data) && isset($data['favs'])) {
$response = $data['favs'];
#!/usr/bin/env php
<?php
require dirname(dirname(__FILE__)) . '/lib/autoload.php';
$command = new Commando\Command();
$command->option()->describedAs('Command to run: default is update.');
$command->option('cache-only')->boolean()->describedAs('Do not load metadata, only use existing cache.');
if ($command[0] === 'termcolor') {
phpterm_demo();
exit;
}
DiscoUtils::log("DiscoJuice update script. Now updating feide feed.", true);
$fav = Favourites::getByID('uuid:d665541c-2fe6-4843-8752-314587f4edd1');
if ($fav === null) {
DiscoUtils::log("No favourites found");
$fav = new Favourites(array('id' => 'uuid:d665541c-2fe6-4843-8752-314587f4edd1', 'favs' => array('foo1', 'foo2', 'bar3')));
$fav->save();
} else {
print_r($fav);
}
DiscoUtils::log("Done.");
function update()
{
$items = array();
$cachefile = '/tmp/feide-services.json';
if (file_exists($cachefile)) {
$services = json_decode(file_get_contents($cachefile), true);
} else {
$url = 'https://api.feide.no/spinfo/0/all';
$data = file_get_contents($url);
file_put_contents($cachefile, $data);
$services = json_decode($data, true);
}
/**
* Load special file based upon Kind ID to entityID from kundeportalen / KIND.
*/
global $BASE;
$mapFile = dirname($BASE) . '/etc/entityIDmapping.json';
$entityIDmapping = json_decode(file_get_contents($mapFile), true);
/**
* Load metadata...
*/
$metadataFile = dirname($BASE) . '/var/dynamic/metadata-prod/saml20-sp-remote.php';
require_once $metadataFile;
$metadataId2entityID = array();
foreach ($metadata as $entityID => $meta) {
if (isset($meta['feide:kindID'])) {
if (!isset($metadataId2entityID[$meta['feide:kindID']])) {
$metadataId2entityID[$meta['feide:kindID']] = array();
}
$metadataId2entityID[$meta['feide:kindID']][] = $entityID;
// echo "entity id " . $meta['feide:kindID'] . ' ' . $entityID . "\n";
}
}
// print_r($metadataId2entityID);
// exit;
$logocache = new LogoCache('feide');
$items = array();
foreach ($services as $k => $d) {
// If we found a mapping from Kind ID to entities based upon metadat, use that
if (isset($metadataId2entityID[$d['id']])) {
$d['entityIDs'] = $metadataId2entityID[$d['id']];
// If not, then use the mapping found in the external file.
} else {
$entityIDs = $this->getEntityIDs($d['id'], $entityIDmapping);
if (!empty($entityIDs)) {
$d['entityIDs'] = $entityIDs;
} else {
continue;
}
foreach ($entityIDs as $entityID) {
if (isset($metadata[$entityID]) && isset($metadata[$entityID]['logo'])) {
// feide/splogos/eu-supply_logo.jpg
if (preg_match('|feide/splogos/(.*)|', $metadata[$entityID]['logo'], $matches)) {
$d['logo'] = $matches[1];
}
}
}
}
// $entityIDhash = sha1(join(',', $entityIDs));
if (isset($d['logo'])) {
// echo " " . $d['id'] . " logo => " . $d['logo'] . "\n";
$src = dirname($BASE) . '/var/dynamic/splogos/' . $d['logo'];
if (!file_exists($src)) {
DiscoUtils::error('Could not find local logo file ' . $src);
} else {
$id = 'feide:sp:' . $d['id'];
$meta = array('type' => 'feide-sp');
if (count($entityIDs) === 1) {
$meta['entityID'] = $entityIDs[0];
}
// $id, $src, $meta, $localFile = false)
$ok = $logocache->getLogoURL($id, $src, $meta, true);
// $ok = $logocache->getLogo($this->entityId, $this->feed, $cl);
if (!empty($ok)) {
$d['icon'] = $ok;
}
}
} else {
DiscoUtils::debug('No logo available for ' . $d['id']);
// echo " " . $d['id'] . " ----- ----- ----- \n";
}
// echo "Dump"; print_r($d);
// if ($i++ > 3) exit;
// print_r($services[$k]);
$item = new FeideService($d);
$items[] = $item;
}
function arrSum($a, $b)
{
$res = array();
foreach ($a as $k => $v) {
if (isset($b[$k]) && isset($a[$k])) {
$res[] = $a[$k] + $b[$k];
} else {
if (isset($a[$k])) {
$res[] = $a[$k];
} else {
$res[] = $b[$k];
}
}
}
return $res;
}
// exit;
$stats = $this->getStatisticsTotal();
$statsfull = $this->getStatsAllWeeks(array_keys($stats));
// print_r($statsfull); exit;
foreach ($items as $item) {
$entityIDs = $item->get('entityIDs');
$total = 0;
foreach ($entityIDs as $entityID) {
if (isset($stats[$entityID])) {
$total += $stats[$entityID];
}
}
$item->set('statistics', $total);
$fullStat = array();
foreach ($entityIDs as $entityID) {
if (isset($statsfull[$entityID])) {
$fullStat = arrSum($statsfull[$entityID], $total);
// $total += $stats[$entityID];
}
}
$item->set('statistics-full', $fullStat);
// $item->save();
// print_r($item); exit;
}
// exit;
foreach ($items as $item) {
$item->save();
// print_r($item); exit;
}
}
function save()
{
$existing = $this->db->{static::$collection}->findOne($this->getQuery());
if ($existing !== null) {
$existingItem = self::fromDB($existing);
if ($this->equalTo($existingItem)) {
DiscoUtils::log('No changes ' . tc_colored('SKIP', 'cyan') . ' ');
} else {
DiscoUtils::log('Object is modified, storing changes ' . tc_colored('UPDATE', 'red') . ' ');
$this->update();
}
} else {
DiscoUtils::log('Metadata is completely new ' . tc_colored('INSERT', 'green') . ' ');
$this->insert();
}
}
protected static function isValidEmbedded($src)
{
if (strpos($src, 'data:image') === 0) {
$splitted = explode(',', $src);
$check = base64_decode($splitted[1]);
if ($check === false) {
DiscoUtils::debug('Skipping logo containing a misformatted embedded logo');
return null;
}
return $check;
} else {
return null;
}
}
public function countryFromURL($entityid)
{
try {
$pu = parse_url($entityid, PHP_URL_HOST);
if (!empty($pu)) {
$rh = strrev($pu);
DiscoUtils::debug('Looking up TLD : ' . $rh);
foreach ($this->countryTLDs as $domain => $country) {
if (DiscoUtils::prefix($domain, $rh)) {
DiscoUtils::debug('Looking up TLD : ' . $rh . ' matched ' . $country);
return $country;
}
}
DiscoUtils::debug('Looking up TLD : ' . $rh . ' DID NOT MATCH any');
}
} catch (Exception $e) {
}
return null;
}
function insertOrUpdateFeed($item)
{
$query = array('id' => $item['id']);
$existing = $this->db->feeds->findOne($query);
if ($existing !== null) {
foreach ($item as $k => $v) {
$existing[$k] = $v;
}
$existing['update'] = new MongoDate();
$this->db->feeds->update($query, $existing);
DiscoUtils::log('Updating feed config ' . tc_colored('UPDATE', 'red') . ' ' . $item['id']);
} else {
$item['created'] = new MongoDate();
$this->db->feeds->insert($item);
DiscoUtils::log('Adding new metadata feed ' . tc_colored('INSERT', 'green') . ' ' . $item['id']);
}
}
#!/usr/bin/env php
<?php
require dirname(dirname(__FILE__)) . '/lib/autoload.php';
$command = new Commando\Command();
$command->option()->describedAs('Command to run: default is update.');
$command->option('cache-only')->boolean()->describedAs('Do not load metadata, only use existing cache.');
if ($command[0] === 'termcolor') {
phpterm_demo();
exit;
}
$backend = new FeideBackend();
if ($command['cache-only']) {
DiscoUtils::log("Running in cache-only mode");
$backend->enableCacheOnly(true);
}
DiscoUtils::log("DiscoJuice update script. Now updating feide feed.", true);
$backend->update();
protected function getLogo(&$data, $m)
{
$logocache = new LogoCache();
// print_r($m);
if (!empty($m['UIInfo']) && !empty($m['UIInfo']['Logo'])) {
$cl = self::getPreferredLogo($m['UIInfo']['Logo']);
// error_log('Preferred logo: ' . var_export($cl, true));
if (!empty($cl)) {
$src = $cl['url'];
$id = sha1($this->feed . '|' . $this->entityId);
$meta = array('entityId' => $this->entityId, 'feed' => $this->feed);
$ok = $logocache->getLogoURL($id, $src, $meta);
// $ok = $logocache->getLogo($this->entityId, $this->feed, $cl);
if (!empty($ok)) {
$data['icon'] = $ok;
}
// $cached = $logocache->getCachedLogo($cl);
// if (!empty($cached)) {
// DiscoUtils::debug("Got logo stored locally: " . $cached);
// // $data['icon'] = $cached;
// }
}
} else {
DiscoUtils::debug('Skipping logo for ' . $data['title']['en'] . ' because it was missing');
}
}