/**
* Method to cleanup the Cron database from old entries.
*/
public static function _CleanupDatabase()
{
$date = new \Core\Date\DateTime();
$date->modify('-2 hours');
$hours = $date->format('U');
$date->modify('-1 day');
$day = $date->format('U');
$date->modify('-1 week');
$week = $date->format('U');
$date->modify('-3 weeks');
$month = $date->format('U');
$date->modify('-5 months');
$half = $date->format('U');
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = 1-minute')->where('created <= ' . $hours)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = 5-minute')->where('created <= ' . $hours)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = 15-minute')->where('created <= ' . $hours)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = 30-minute')->where('created <= ' . $hours)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = hourly')->where('created <= ' . $week)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = 2-hour')->where('created <= ' . $week)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = 3-hour')->where('created <= ' . $week)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = 6-hour')->where('created <= ' . $week)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = 12-hour')->where('created <= ' . $week)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = daily')->where('created <= ' . $week)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = weekly')->where('created <= ' . $month)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('cron_log')->where('cron = monthly')->where('created <= ' . $half)->execute();
return true;
}
/**
* Test that the Execute method properly handles a generic Dataset object.
*/
public function testExecute(){
$ds = \Core\Datamodel\Dataset::Init()
->select('*')
->table('component');
$this->dmi->execute($ds);
$this->assertGreaterThan(0, sizeof($ds->num_rows));
}
public function index()
{
$request = $this->getPageRequest();
$view = $this->getView();
$isAdmin = \Core\user()->checkAccess('g:admin');
$serverid = isset($_SERVER['HTTP_X_CORE_SERVER_ID']) ? $_SERVER['HTTP_X_CORE_SERVER_ID'] : null;
// If the server ID is set, it should be a 32-digit character.
// Anything else and omit.
if (strlen($serverid) != 32) {
$serverid = null;
} elseif (!preg_match('/^[A-Z0-9]*$/', $serverid)) {
// Invalid string.
$serverid = null;
}
$ua = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';
if (strpos($ua, '(http://corepl.us)') !== false) {
/** @var string $uav ex: "Core Plus 1.2.3" */
$uav = str_replace(' (http://corepl.us)', '', $ua);
/** @var string $version Just the version, ex: "1.2.3" */
$version = str_replace('Core Plus ', '', $uav);
// The set of logic to compare the current version of Core against the version connecting.
// This is used primarily to set a class name onto the graphs so that they can be coloured specifically.
$v = Core::VersionSplit($version);
// These two values are used in the historical map, (as revision may be a bit useless at this scale).
$briefVersion = $v['major'] . '.' . $v['minor'];
} elseif ($request->getParameter('packager')) {
$briefVersion = $request->getParameter('packager');
} else {
$briefVersion = null;
}
// Record this key as connected.
if ($serverid) {
$licmod = PackageRepositoryLicenseModel::Construct($serverid);
$licmod->set('datetime_last_checkin', Core\Date\DateTime::NowGMT());
$licmod->set('ip_last_checkin', REMOTE_IP);
$licmod->set('referrer_last_checkin', isset($_SERVER['HTTP_REFERER']) ? $_SERVER['HTTP_REFERER'] : '');
$licmod->set('useragent_last_checkin', isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '');
$licmod->save();
}
if ($request->ctype == 'application/xml') {
// This is a repo.xml request, usually for debugging purposes, (as the app requests gz compressed versions).
$xml = PackageRepositoryPackageModel::GetAsRepoXML($serverid, $briefVersion);
$view->mode = View::MODE_NOOUTPUT;
$view->contenttype = 'application/xml';
$view->render();
echo $xml->asMinifiedXML();
return;
} elseif ($request->ext == 'xml.gz') {
// This is a normal from-client request; a compressed repo.xml structure.
$xml = PackageRepositoryPackageModel::GetAsRepoXML($serverid, $briefVersion);
$view->mode = View::MODE_NOOUTPUT;
$view->contenttype = 'application/gzip';
$view->render();
$c = $xml->asMinifiedXML();
echo gzencode($c);
return;
} else {
// This is a browse request.
$packages = [];
$where = [];
if ($briefVersion) {
$where[] = 'packager LIKE ' . $briefVersion . '%';
}
$allPackages = PackageRepositoryPackageModel::Find($where);
foreach ($allPackages as $pkg) {
/** @var PackageRepositoryPackageModel $pkg */
$pkgKey = $pkg->get('type') . '-' . $pkg->get('key');
$pkgPackager = $pkg->get('packager');
$pkgVersion = $pkg->get('version');
if (!isset($packages[$pkgKey])) {
$packages[$pkgKey] = ['package' => $pkg, 'check' => $pkgVersion, 'min' => $pkgPackager, 'max' => $pkgPackager];
}
if (\Core\version_compare($pkgPackager, $packages[$pkgKey]['min'], 'lt')) {
// Record the lowest supported Core version for this package
$packages[$pkgKey]['min'] = $pkgPackager;
}
if (\Core\version_compare($pkgPackager, $packages[$pkgKey]['max'], 'gt')) {
// Record the higest supported Core version for this package
$packages[$pkgKey]['max'] = $pkgPackager;
}
if (\Core\version_compare($pkgVersion, $packages[$pkgKey]['check'], 'gt')) {
// Save the newest component as the reference package for all the display data.
$packages[$pkgKey]['check'] = $pkgVersion;
$packages[$pkgKey]['package'] = $pkg;
}
}
// Build a list of all supported versions of Core in the database.
$allCoreVersions = [];
$packagerVersions = \Core\Datamodel\Dataset::Init()->unique(true)->select('packager')->table('package_repository_package')->order('packager')->executeAndGet();
foreach ($packagerVersions as $pkg) {
$pkgObject = new \Core\VersionString($pkg);
$pkgBase = $pkgObject->major . '.' . $pkgObject->minor;
if (!isset($allCoreVersions[$pkgBase])) {
$allCoreVersions[$pkgBase] = 'Core Plus ' . $pkgBase;
}
}
krsort($allCoreVersions, SORT_NATURAL);
$allCoreVersions = array_merge(['' => '-- All Versions --'], $allCoreVersions);
$versionSelector = new Form();
$versionSelector->set('method', 'get');
$versionSelector->addElement('select', ['name' => 'packager', 'options' => $allCoreVersions, 'value' => $briefVersion]);
$versionSelector->addElement('submit', ['value' => 'Filter']);
$view->assign('version_selector', $versionSelector);
$view->assign('packages', $packages);
$view->assign('version_selected', $briefVersion);
}
if ($isAdmin) {
$view->addControl(['title' => t('STRING_PACKAGE_REPOSITORY_REBUILD'), 'link' => '/packagerepository/rebuild']);
}
$view->title = 'Package Repository';
$view->templatename = 'pages/packagerepository/index.tpl';
$view->assign('is_admin', $isAdmin);
}
/**
* This will check and see how many 404 requests there have been recently.
*
* @return bool
*/
public static function Check404Pages()
{
// How long back do I want to check the logs?
$time = new DateTime();
$time->modify('-30 seconds');
$ds = Dataset::Init()->table('user_activity')->where(['status = 404', 'ip_addr = ' . REMOTE_IP, 'datetime > ' . $time->format('U')])->count()->execute();
if ($ds->num_rows > 30) {
// CHILL THAR F****R!
$time->modify('+6 minutes');
$blacklist = new \IpBlacklistModel();
$blacklist->setFromArray(['ip_addr' => REMOTE_IP . '/24', 'expires' => $time->format('U'), 'message' => 'You have requested too many "404" pages recently, please go get some coffee and wait for a short bit. If you are a bot and/or spammer, please bugger off.', 'comment' => '5-minute auto-ban for too many 404 requests in 30 seconds']);
$blacklist->save();
\SystemLogModel::LogSecurityEvent('/security/blocked', 'Blocking IP due to too many 404 requests in 30 seconds.');
die($blacklist->get('message'));
}
}
<?php
/**
* Upgrade file for migrating user data from the user user configs to the new supplemental user table.
*/
// The migrations for source (user_user_config) to destination (user).
$migrations = ['first_name' => 'first_name', 'last_name' => 'last_name', 'phone' => 'phone', 'bio' => 'bio', 'username' => 'username'];
// Lookup each key and the corresponding user attached.
$sources = \Core\Datamodel\Dataset::Init()->select(['user_id', 'key', 'value'])->table('user_user_config')->where('key IN ' . implode(',', array_keys($migrations)))->executeAndGet();
$users = [];
foreach ($sources as $s) {
if (!isset($users[$s['user_id']])) {
$users[$s['user_id']] = UserModel::Construct($s['user_id']);
}
/** @var UserModel $u */
$u = $users[$s['user_id']];
$nk = $migrations[$s['key']];
$nv = $s['value'];
$u->set($nk, $nv);
}
// Now save each user loaded into the array.
foreach ($users as $u) {
/** @var UserModel $u */
$u->save();
}
/**
* Display a list of system logs that have been recorded.
*
* @return int
*/
public function log(){
$view = $this->getView();
$request = $this->getPageRequest();
if(!\Core\user()->checkAccess('p:/core/systemlog/view')){
return View::ERROR_ACCESSDENIED;
}
$codes = ['' => '-- All --'];
$ds = Dataset::Init()
->table('system_log')
->select('code')
->unique(true)
->order('code')
->execute();
foreach($ds as $row){
$codes[$row['code']] = $row['code'];
}
$listings = new Core\ListingTable\Table();
$listings->setModelName('SystemLogModel');
$listings->setName('system-log');
$listings->setDefaultSort('datetime');
$listings->addFilter(
'select',
array(
'title' => 'Type',
'name' => 'type',
'options' => array(
'' => '-- All --',
'info' => 'Informative',
'error' => 'Warning/Error',
'security' => 'Security',
),
'link' => FilterForm::LINK_TYPE_STANDARD,
)
);
$listings->addFilter(
'select',
[
'title' => 'Code',
'name' => 'code',
'options' => $codes,
'link' => FilterForm::LINK_TYPE_STANDARD,
]
);
$listings->addFilter(
'date',
[
'title' => 'On or After',
'name' => 'datetime_onafter',
'linkname' => 'datetime',
'link' => FilterForm::LINK_TYPE_GE,
]
);
$listings->addFilter(
'date',
[
'title' => 'On or Before',
'name' => 'datetime_onbefore',
'linkname' => 'datetime',
'link' => FilterForm::LINK_TYPE_LE,
'linkvaluesuffix' => ' 23:59:59'
]
);
/*$filters->addElement(
'select',
array(
'title' => 'Cron',
'name' => 'cron',
'options' => array(
'' => '-- All --',
'hourly' => 'hourly',
'daily' => 'daily',
'weekly' => 'weekly',
'monthly' => 'monthly'
),
'link' => FilterForm::LINK_TYPE_STANDARD,
)
);
$filters->addElement(
'select',
array(
'title' => 'Status',
'name' => 'status',
'options' => array(
'' => '-- All --',
'pass' => 'pass',
'fail' => 'fail'
),
'link' => FilterForm::LINK_TYPE_STANDARD,
)
);*/
$listings->addFilter(
'hidden',
array(
'title' => 'Session',
'name' => 'session_id',
'link' => FilterForm::LINK_TYPE_STANDARD,
)
);
$listings->addFilter(
'user',
array(
'title' => 'User',
'name' => 'affected_user_id',
'linkname' => [
'affected_user_id',
'user_id',
]
//'link' => FilterForm::LINK_TYPE_STANDARD,
)
);
$listings->addFilter(
'text',
[
'title' => 'IP Address',
'name' => 'ip_addr',
'link' => FilterForm::LINK_TYPE_STARTSWITH,
]
);
$listings->addColumn('Type', 'type');
$listings->addColumn('Date/Time', 'datetime');
$listings->addColumn('IP Address', 'ip_addr');
$listings->addColumn('Message', 'message');
//$listings->addColumn('Session', 'session_id');
$listings->addColumn('User', 'user_id', false);
$listings->addColumn('Affected User', 'affected_user_id', false);
$listings->loadFiltersFromRequest($request);
$view->mastertemplate = 'admin';
$view->title = 't:STRING_SYSTEM_LOG';
$view->assign('listings', $listings);
}
CLI::PrintHeader('Cleaning up non-existent pages');
$pageremovecount = 0;
foreach(\Core\Datamodel\Dataset::Init()->select('baseurl')->table('page')->where('admin = 1')->execute() as $row){
$baseurl = $row['baseurl'];
// This page existed already, no need to do anything :)
if(isset($allpages[$baseurl])) continue;
++$pageremovecount;
// Otherwise, this page was deleted or for some reason doesn't exist in the component list.....
// BUH BAI
CLI::PrintActionStart('Deleting page ' . $baseurl);
$changes[] = "Deleted page " . $baseurl;
\Core\Datamodel\Dataset::Init()->delete()->table('page')->where('baseurl = ' . $baseurl)->execute();
\Core\Datamodel\Dataset::Init()->delete()->table('page_meta')->where('baseurl = ' . $baseurl)->execute();
CLI::PrintActionStatus('ok');
}
if($pageremovecount == 0){
CLI::PrintLine('No pages flushed');
}
CLI::PrintHeader('Synchronizing Searchable Models');
foreach(\Core::GetComponents() as $c){
/** @var Component_2_1 $c */
foreach($c->getClassList() as $class => $file){
if($class == 'model'){
continue;
}
if(strrpos($class, 'model') !== strlen($class) - 5){
/**
* View controller for a blog article listing page.
* This will only display articles under this same blog.
*
* @param BlogModel $blog
*/
private function _viewBlog(BlogModel $blog)
{
$view = $this->getView();
$page = $blog->getLink('Page');
$request = $this->getPageRequest();
$manager = \Core\user()->checkAccess('p:/blog/manage_all');
$editor = \Core\user()->checkAccess($blog->get('manage_articles_permission ')) || $manager;
$viewer = \Core\user()->checkAccess($blog->get('access')) || $editor;
// Get the latest published article's update date. This will be used for the blog updated timestamp.
// (This doesn't have a whole lot of benefit above the ModelFactory, simply illustrating a different way to query data).
$latest = \Core\Datamodel\Dataset::Init()->select('*')->table('page')->where('parenturl = ' . $blog->get('baseurl'))->where('published_status = published')->order('published DESC')->limit(1)->current();
$filters = new FilterForm();
$filters->haspagination = true;
// Allow different type of requests to come in here.
switch ($request->ctype) {
case 'application/atom+xml':
$view->templatename = 'pages/blog/view-blog.atom.tpl';
$view->contenttype = $request->ctype;
$view->mastertemplate = false;
$filters->setLimit(200);
break;
case 'application/rss+xml':
$view->templatename = 'pages/blog/view-blog.rss.tpl';
$view->contenttype = $request->ctype;
$view->mastertemplate = false;
$filters->setLimit(200);
break;
default:
$view->templatename = 'pages/blog/view-blog.tpl';
$filters->setLimit(20);
break;
}
$filters->load($this->getPageRequest());
$factory = new ModelFactory('PageModel');
if ($request->getParameter('q')) {
$query = $request->getParameter('q');
$factory->where(\Core\Search\Helper::GetWhereClause($request->getParameter('q')));
} else {
$query = null;
}
$factory->where('parenturl = ' . $blog->get('baseurl'));
$factory->order('published DESC');
if (!$editor) {
// Limit these to published articles.
$factory->where('published_status = published');
// And where the published date is >= now.
$factory->where('published <= ' . CoreDateTime::Now('U', Time::TIMEZONE_GMT));
}
$filters->applyToFactory($factory);
$articles = $factory->get();
$view->mode = View::MODE_PAGEORAJAX;
$view->assign('blog', $blog);
$view->assign('articles', $articles);
$view->assign('page', $page);
$view->assign('filters', $filters);
$view->assign('canonical_url', \Core\resolve_link($blog->get('baseurl')));
$view->assign('last_updated', $latest ? $latest['updated'] : 0);
$view->assign('servername', SERVERNAME_NOSSL);
$view->assign('editor', $editor);
$view->assign('add_article_link', '/content/create?page_template=blog-article.tpl&parenturl=' . $blog->get('baseurl'));
// Add the extra view types for this page
$view->addHead('<link rel="alternate" type="application/atom+xml" title="' . $page->get('title') . ' Atom Feed" href="' . \Core\resolve_link($blog->get('baseurl')) . '.atom"/>');
$view->addHead('<link rel="alternate" type="application/rss+xml" title="' . $page->get('title') . ' RSS Feed" href="' . \Core\resolve_link($blog->get('baseurl')) . '.rss"/>');
if ($editor) {
if ($blog->get('type') == 'remote') {
$view->addControl('Import Feed', '/blog/import/' . $blog->get('id'), 'exchange');
} else {
$view->addControl('Add Article', '/content/create?page_template=blog-article.tpl&parenturl=' . $blog->get('baseurl'), 'add');
}
}
if ($manager) {
$view->addControl('Edit Blog', '/blog/update/' . $blog->get('id'), 'edit');
$view->addControl('All Articles', '/admin/pages/?filter[parenturl]=' . $blog->get('baseurl'), 'tasks');
}
$view->addControl('RSS Feed', \Core\resolve_link($blog->get('baseurl')) . '.rss', 'rss');
//$view->addControl('Atom Feed', \Core\resolve_link($blog->get('baseurl')) . '.atom', 'rss');
}
$fac->where('groups != ');
while(($user = $fac->getNext())){
/** @var UserModel $user */
// datamodel backed users have the groups listed in their column "groups".
$g = json_decode($user->get('groups'), true);
if(!$g) $g = array();
$gids = [];
foreach($g as $gid){
if(is_array($gid)){
$gids = array_merge($gids, $gid);
}
else{
$gids[] = $gid;
}
}
$gids = array_unique($gids);
$user->setGroups($gids);
$user->save();
}
// Now wipe out this temporary column data.
\Core\Datamodel\Dataset::Init()
->update(['groups' => ''])
->table('user')
->where('groups != ')
->execute();
/**
* Extract all possible keywords from the content along with their current scores.
*
* @return array
*/
public function getKeywords()
{
$s = strtolower($this->content);
$s = strip_tags($s);
$s = preg_replace('/[^a-z0-9 ]/', '', $s);
$stopwords = \Core\get_stop_words();
// Add on a few more custom stop words that don't necessarily belong in upstream.
$stopwords[] = 'id';
$stopwords[] = 'like';
$exploded = explode(' ', $s);
$nt = [];
foreach ($exploded as $i => $w) {
if ($w == '') {
continue;
}
if (in_array($w, $stopwords)) {
continue;
}
$nt[] = $w;
if (isset($exploded[$i + 1])) {
$nt[] = $w . ' ' . $exploded[$i + 1];
}
if (isset($exploded[$i + 2])) {
$nt[] = $w . ' ' . $exploded[$i + 1] . ' ' . $exploded[$i + 2];
}
}
$nt = array_unique($nt);
sort($nt);
$all = Dataset::Init()->select('*')->table('spam_ham_keyword')->execute();
// Convert this to something I can quickly check through.
$keywords = [];
foreach ($all as $row) {
$keywords[$row['keyword']] = $row['score'];
}
$ret = [];
foreach ($nt as $w) {
if (isset($keywords[$w])) {
$score = $keywords[$w];
} else {
$score = 0;
}
$ret[] = ['keyword' => $w, 'score' => $score];
}
return $ret;
}
<?php
/**
* Upgrade file for migrating user data from the user user configs to the new supplemental user table.
*/
// The migrations for source (user_user_config) to destination (user).
$migrations = [
'/user/gpgauth/pubkey' => 'gpgauth_pubkey',
'json:profiles' => 'external_profiles',
];
// Lookup each key and the corresponding user attached.
$sources = \Core\Datamodel\Dataset::Init()
->select(['user_id', 'key', 'value'])
->table('user_user_config')
->where('key IN ' . implode(',', array_keys($migrations)))
->executeAndGet();
$users = [];
foreach($sources as $s){
if(!isset($users[ $s['user_id'] ])){
$users[ $s['user_id'] ] = UserModel::Construct($s['user_id']);
}
/** @var UserModel $u */
$u = $users[ $s['user_id'] ];
$nk = $migrations[ $s['key'] ];
$nv = $s['value'];
$u->set($nk, $nv);
