public function board_statistics($output, $shortname = null) { $boards = $this->radix_coll->getAll(); $available = $this->board_stats->getAvailableStats(); while (true) { // Obtain all of the statistics already stored on the database to check for update frequency. $stats = $this->dc->qb()->select('board_id, name, timestamp')->from($this->dc->p('plugin_fu_board_statistics'), 'bs')->orderBy('timestamp', 'desc')->execute()->fetchAll(); // Obtain the list of all statistics enabled. $avail = []; foreach ($available as $k => $a) { // get only the non-realtime ones if (isset($available['frequency'])) { $avail[] = $k; } } foreach ($boards as $board) { if (!is_null($shortname) && $shortname != $board->shortname) { continue; } // Update all statistics for the specified board or current board. $output->writeln($board->shortname . ' (' . $board->id . ')'); foreach ($available as $k => $a) { $output->writeln(' ' . $k . ' '); $found = false; $skip = false; foreach ($stats as $r) { // Determine if the statistics already exists or that the information is outdated. if ($r['board_id'] == $board->id && $r['name'] == $k) { // This statistics report has run once already. $found = true; if (!isset($a['frequency'])) { $skip = true; continue; } // This statistics report has not reached its frequency EOL. if (time() - $r['timestamp'] <= $a['frequency']) { $skip = true; continue; } break; } } // racing conditions with our cron. if ($found === false) { $this->board_stats->saveStat($board->id, $k, time() + 600, ''); } // We were able to obtain a LOCK on the statistics report and has already reached the // targeted frequency time. if ($skip === false) { $output->writeln('* Processing...'); $process = 'process' . $a['function']; $result = $this->board_stats->{$process}($board); // Save the statistics report in a JSON array. $this->board_stats->saveStat($board->id, $k, time(), $result); } } } sleep(10); } }
/** * Adds a new ban * * @param string $ip_decimal The IP of the banned user in decimal format * @param string $reason The reason for the ban * @param int $length The length of the ban in seconds * @param array $board_ids The array of board IDs, global ban if left empty * * @return \Foolz\Foolslide\Model\Ban * @throws \Foolz\Foolslide\Model\BanException */ public function add($ip_decimal, $reason, $length, $board_ids = []) { // 0 is a global ban if (empty($board_ids)) { $board_ids = [0]; } else { // check that all ids are existing boards $valid_board_ids = []; foreach ($this->radix_coll->getAll() as $board) { $valid_board_ids[] = $board->id; } foreach ($board_ids as $id) { if (!in_array($id, $valid_board_ids)) { throw new BanException(_i('You entered a non-existent board ID.')); } } } if (!ctype_digit((string) $ip_decimal)) { throw new BanException(_i('You entered an invalid IP.')); } if (mb_strlen($reason, 'utf-8') > 10000) { throw new BanException(_i('You entered a too long reason for the ban.')); } if (!ctype_digit($length)) { throw new BanException(_i('You entered an invalid length for the ban.')); } $time = time(); $objects = []; try { $old = $this->getByIp($ip_decimal); } catch (BanNotFoundException $e) { $old = false; } foreach ($board_ids as $board_id) { $new = new Ban($this->getContext()); $new->ip = $ip_decimal; $new->reason = $reason; $new->start = $time; $new->length = $length; $new->board_id = $board_id; $new->creator_id = $this->getAuth()->getUser()->getId(); $new->appeal = ''; if (isset($old[$new->board_id])) { if ($new->length < $old[$new->board_id]->length) { $new->length = $old[$new->board_id]->length; } $this->dc->qb()->update($this->dc->p('banned_posters'))->where('id = :id')->set('start', $new->start)->set('length', $new->length)->set('creator_id', $new->creator_id)->setParameter(':id', $old[$new->board_id]->id)->execute(); } else { $this->dc->getConnection()->insert($this->dc->p('banned_posters'), ['ip' => $new->ip, 'reason' => $new->reason, 'start' => $new->start, 'length' => $new->length, 'board_id' => $new->board_id, 'creator_id' => $new->creator_id, 'appeal' => $new->appeal]); } $objects[] = $new; } return $objects; }
public function action_sphinx_config() { $data = []; $mysql = $this->preferences->get('foolslide.sphinx.listen_mysql', null); $data['mysql'] = ['host' => $mysql === null ? '127.0.0.1' : explode(':', $mysql)[0], 'port' => $mysql === null ? '3306' : explode(':', $mysql)[1], 'flag' => $this->preferences->get('foolslide.sphinx.connection_flags', '0')]; $sphinx = $this->preferences->get('foolslide.sphinx.listen', null); $data['sphinx'] = ['port' => $sphinx === null ? '9306' : explode(':', $sphinx)[1], 'working_directory' => $this->preferences->get('foolslide.sphinx.dir', '/usr/local/sphinx'), 'mem_limit' => $this->preferences->get('foolslide.sphinx.mem_limit', '1024M'), 'min_word_len' => $this->preferences->get('foolslide.sphinx.min_word_len', 1), 'max_children' => $this->preferences->get('foolslide.sphinx.max_children', 0), 'max_matches' => $this->preferences->get('foolslide.sphinx.max_matches', 5000), 'distributed' => (int) $this->preferences->get('foolslide.sphinx.distributed', 0)]; $data['boards'] = $this->radix_coll->getAll(); $data['example'] = current($data['boards']); $this->param_manager->setParam('method_title', [_i('Search'), 'Sphinx', _i('Configuration File'), _i('Generate')]); $this->builder->createPartial('body', $data['sphinx']['distributed'] > 1 ? 'boards/sphinx_dist_config' : 'boards/sphinx_config')->getParamManager()->setParams($data); return new Response($this->builder->build()); }
/** * Gets the search results * * @return \Foolz\Foolslide\Model\Search The current object * @throws SearchEmptyResultException If there's no results to display * @throws SearchRequiresSphinxException If the search submitted requires Sphinx to run * @throws SearchSphinxOfflineException If the Sphinx server is unreachable * @throws SearchInvalidException If the values of the search weren't compatible with the domain */ protected function p_getSearchComments() { $this->profiler->log('Board::getSearchComments Start'); extract($this->options); // set all empty fields to null $search_fields = ['boards', 'subject', 'text', 'username', 'tripcode', 'email', 'capcode', 'uid', 'poster_ip', 'filename', 'image', 'deleted', 'ghost', 'filter', 'type', 'start', 'end', 'results', 'order']; foreach ($search_fields as $field) { if (!isset($args[$field])) { $args[$field] = null; } } // populate an array containing all boards that would be searched $boards = []; if ($args['boards'] !== null) { foreach ($args['boards'] as $board) { $b = $this->radix_coll->getByShortname($board); if ($b) { $boards[] = $b; } } } // search all boards if none selected if (count($boards) == 0) { $boards = $this->radix_coll->getAll(); } // if image is set, get either the media_hash or media_id if ($args['image'] !== null) { if (substr($args['image'], -2) !== '==') { $args['image'] .= '=='; } // if board is set, retrieve media_id if ($this->radix !== null) { try { $media = $this->media_factory->getByMediaHash($this->radix, $args['image']); } catch (MediaNotFoundException $e) { $this->comments_unsorted = []; $this->comments = []; $this->profiler->log('Board::getSearchComments Ended Prematurely'); throw new SearchEmptyResultException(_i('No results found.')); } $args['image'] = $media->media_id; } } if ($this->radix === null && !$this->preferences->get('foolslide.sphinx.global')) { // global search requires sphinx throw new SearchRequiresSphinxException(_i('Sorry, this action requires the Sphinx to be installed and running.')); } elseif ($this->radix === null && $this->preferences->get('foolslide.sphinx.global') || $this->radix !== null && $this->radix->sphinx) { // configure sphinx connection params $sphinx = explode(':', $this->preferences->get('foolslide.sphinx.listen')); $conn = new SphinxConnnection(); $conn->setParams(['host' => $sphinx[0], 'port' => $sphinx[1], 'options' => [MYSQLI_OPT_CONNECT_TIMEOUT => 5]]); $conn->silenceConnectionWarning(true); // determine if all boards will be used for search or not if ($this->radix == null) { $indexes = []; foreach ($boards as $radix) { if (!$radix->sphinx) { continue; } $indexes[] = $radix->shortname . '_ancient'; $indexes[] = $radix->shortname . '_main'; $indexes[] = $radix->shortname . '_delta'; } } else { $indexes = [$this->radix->shortname . '_ancient', $this->radix->shortname . '_main', $this->radix->shortname . '_delta']; } // establish connection try { $query = SphinxQL::create($conn)->select('id', 'board')->from($indexes); } catch (\Foolz\SphinxQL\ConnectionException $e) { throw new SearchSphinxOfflineException(_i('The search backend is currently unavailable.')); } // parse search params if ($args['subject'] !== null) { $query->match('title', $args['subject']); } if ($args['text'] !== null) { if (mb_strlen($args['text'], 'utf-8') < 1) { return []; } $query->match('comment', $args['text'], true); } if ($args['username'] !== null) { $query->match('name', $args['username']); } if ($args['tripcode'] !== null) { $query->match('trip', '"' . $args['tripcode'] . '"'); } if ($args['email'] !== null) { $query->match('email', $args['email']); } if ($args['capcode'] !== null) { if ($args['capcode'] === 'user') { $query->where('cap', ord('N')); } elseif ($args['capcode'] === 'mod') { $query->where('cap', ord('M')); } elseif ($args['capcode'] === 'admin') { $query->where('cap', ord('A')); } elseif ($args['capcode'] === 'dev') { $query->where('cap', ord('D')); } } if ($args['uid'] !== null) { $query->match('pid', $args['uid']); } if ($this->getAuth()->hasAccess('comment.see_ip') && $args['poster_ip'] !== null) { $query->where('pip', (int) Inet::ptod($args['poster_ip'])); } if ($args['filename'] !== null) { $query->match('media_filename', $args['filename']); } if ($args['image'] !== null) { if ($this->radix !== null) { $query->where('mid', (int) $args['image']); } else { $query->match('media_hash', '"' . $args['image'] . '"'); } } if ($args['deleted'] !== null) { if ($args['deleted'] == 'deleted') { $query->where('is_deleted', 1); } if ($args['deleted'] == 'not-deleted') { $query->where('is_deleted', 0); } } if ($args['ghost'] !== null) { if ($args['ghost'] == 'only') { $query->where('is_internal', 1); } if ($args['ghost'] == 'none') { $query->where('is_internal', 0); } } if ($args['filter'] !== null) { if ($args['filter'] == 'image') { $query->where('has_image', 0); } if ($args['filter'] == 'text') { $query->where('has_image', 1); } } if ($args['type'] !== null) { if ($args['type'] == 'sticky') { $query->where('is_sticky', 1); } if ($args['type'] == 'op') { $query->where('is_op', 1); } if ($args['type'] == 'posts') { $query->where('is_op', 0); } } if ($args['start'] !== null) { $query->where('timestamp', '>=', intval(strtotime($args['start']))); } if ($args['end'] !== null) { $query->where('timestamp', '<=', intval(strtotime($args['end']))); } if ($args['results'] !== null) { if ($args['results'] == 'op') { $query->groupBy('thread_num'); $query->withinGroupOrderBy('is_op', 'desc'); } if ($args['results'] == 'posts') { $query->where('is_op', 0); } } if ($args['order'] !== null && $args['order'] == 'asc') { $query->orderBy('timestamp', 'ASC'); } else { $query->orderBy('timestamp', 'DESC'); } $max_matches = $this->preferences->get('foolslide.sphinx.max_matches', 5000); // set sphinx options $query->limit($limit)->offset($page * $limit - $limit >= $max_matches ? $max_matches - 1 : $page * $limit - $limit)->option('max_matches', (int) $max_matches)->option('reverse_scan', $args['order'] === 'asc' ? 0 : 1); // submit query try { $search = $query->execute(); } catch (\Foolz\SphinxQL\DatabaseException $e) { $this->logger->error('Search Error: ' . $e->getMessage()); throw new SearchInvalidException(_i('The search backend returned an error.')); } // no results found if (!count($search)) { $this->comments_unsorted = []; $this->comments = []; throw new SearchEmptyResultException(_i('No results found.')); } $sphinx_meta = Helper::pairsToAssoc(Helper::create($conn)->showMeta()->execute()); $this->total_count = $sphinx_meta['total']; $this->total_found = $sphinx_meta['total_found']; // populate sql array for full records $sql = []; foreach ($search as $doc => $result) { $board = $this->radix_coll->getById($result['board']); $sql[] = $this->dc->qb()->select('*, ' . $result['board'] . ' AS board_id')->from($board->getTable(), 'r')->leftJoin('r', $board->getTable('_images'), 'mg', 'mg.media_id = r.media_id')->where('doc_id = ' . $this->dc->getConnection()->quote($result['id']))->getSQL(); } $result = $this->dc->getConnection()->executeQuery(implode(' UNION ', $sql))->fetchAll(); } else { // this is not implemented yet, would require some sort of MySQL search throw new SearchRequiresSphinxException(_i('Sorry, this board does not have search enabled.')); } // no results found IN DATABASE, but we might still get a search count from Sphinx if (!count($result)) { $this->comments_unsorted = []; $this->comments = []; } else { // process results foreach ($result as $key => $row) { $board = $this->radix !== null ? $this->radix : $this->radix_coll->getById($row['board_id']); $bulk = new CommentBulk(); $bulk->import($row, $board); $this->comments_unsorted[] = $bulk; unset($result[$key]); } } $this->comments[0]['posts'] = $this->comments_unsorted; return $this; }
/** * Bans an image for a board or for all the boards * * @param boolean $global False if the media should be banned only on current Radix, true if it should be banned on all the Radix and future ones */ public function p_ban($global = false) { if ($global === false) { $this->dc->qb()->update($this->radix->getTable('_images'))->set('banned', 1)->where('media_id = :media_id')->setParameter(':media_id', $this->media->media_id)->execute(); $this->delete(true, true, true); return; } $result = $this->dc->qb()->select('COUNT(*) as count')->from($this->radix->getTable('_images'), 'ri')->where('media_hash = :md5')->setParameter(':md5', $this->media->media_hash)->execute()->fetch(); if (!$result['count']) { $this->dc->getConnection()->insert('banned_md5', ['md5' => $this->media->media_hash])->execute(); } foreach ($this->radix_coll->getAll() as $radix) { try { $media = $this->media_factory->getByMediaHash($radix, $this->media->media_hash); $media = new Media($this->getContext(), CommentBulk::forge($this->radix, null, $media)); $this->dc->qb()->update($radix->getTable('_images'))->set('banned', 1)->where('media_id = :media_id')->setParameter(':media_id', $media->media_id)->execute(); $media->delete(true, true, true); } catch (MediaNotFoundException $e) { $this->dc->getConnection()->insert($radix->getTable('_images'), ['media_hash' => $this->media->media_hash, 'banned' => 1]); } } }
public function radix_search() { if ($this->getPost('submit_search_global')) { $this->radix = null; } $text = $this->getPost('text'); if ($this->radix !== null && $this->getPost('submit_post')) { return $this->radix_post(str_replace(',', '_', $text)); } $this->response = new StreamedResponse(); // Check all allowed search modifiers and apply only these $modifiers = ['boards', 'subject', 'text', 'username', 'tripcode', 'email', 'filename', 'capcode', 'uid', 'image', 'deleted', 'ghost', 'type', 'filter', 'start', 'end', 'order', 'page']; if ($this->getAuth()->hasAccess('comment.see_ip')) { $modifiers[] = 'poster_ip'; $modifiers[] = 'deletion_mode'; } // GET -> URL Redirection to provide URL presentable for sharing links. if ($this->getPost()) { if ($this->radix !== null) { $redirect_url = [$this->radix->shortname, 'search']; } else { $redirect_url = ['_', 'search']; } foreach ($modifiers as $modifier) { if ($this->getPost($modifier)) { if ($modifier === 'image') { array_push($redirect_url, $modifier); array_push($redirect_url, rawurlencode(Media::urlsafe_b64encode(Media::urlsafe_b64decode($this->getPost($modifier))))); } elseif ($modifier === 'boards') { if ($this->getPost('submit_search_global')) { } elseif (count($this->getPost($modifier)) == 1) { $boards = $this->getPost($modifier); $redirect_url[0] = $boards[0]; } elseif (count($this->getPost($modifier)) > 1) { $redirect_url[0] = '_'; // avoid setting this if we're just searching on all the boards $sphinx_boards = []; foreach ($this->radix_coll->getAll() as $k => $b) { if ($b->sphinx) { $sphinx_boards[] = $b; } } if (count($sphinx_boards) !== count($this->getPost($modifier))) { array_push($redirect_url, $modifier); array_push($redirect_url, rawurlencode(implode('.', $this->getPost($modifier)))); } } } else { array_push($redirect_url, $modifier); array_push($redirect_url, rawurlencode($this->getPost($modifier))); } } } return new RedirectResponse($this->uri->create($redirect_url), 303); } $search = $this->uri->uri_to_assoc($this->request->getPathInfo(), 1, $modifiers); $this->param_manager->setParam('search', $search); // latest searches system if (!is_array($cookie_array = @json_decode($this->getCookie('search_latest_5'), true))) { $cookie_array = []; } // sanitize foreach ($cookie_array as $item) { // all subitems must be array, all must have 'board' if (!is_array($item) || !isset($item['board'])) { $cookie_array = []; break; } } $search_opts = array_filter($search); $search_opts['board'] = $this->radix !== null ? $this->radix->shortname : false; unset($search_opts['page']); // if it's already in the latest searches, remove the previous entry foreach ($cookie_array as $key => $item) { if ($item === $search_opts) { unset($cookie_array[$key]); break; } } // we don't want more than 5 entries for latest searches if (count($cookie_array) > 4) { array_pop($cookie_array); } array_unshift($cookie_array, $search_opts); $this->builder->getPartial('tools_search')->getParamManager()->setParam('latest_searches', $cookie_array); $this->response->headers->setCookie(new Cookie($this->getContext(), 'search_latest_5', json_encode($cookie_array), 60 * 60 * 24 * 30)); foreach ($search as $key => $value) { if ($value !== null) { $search[$key] = trim(rawurldecode($value)); } } if ($search['boards'] !== null) { $search['boards'] = explode('.', $search['boards']); } if ($search['image'] !== null) { $search['image'] = base64_encode(Media::urlsafe_b64decode($search['image'])); } if ($this->getAuth()->hasAccess('comment.see_ip') && $search['poster_ip'] !== null) { if (!filter_var($search['poster_ip'], FILTER_VALIDATE_IP)) { return $this->error(_i('The poster IP you inserted is not a valid IP address.')); } $search['poster_ip'] = Inet::ptod($search['poster_ip']); } try { $board = Search::forge($this->getContext())->getSearch($search)->setRadix($this->radix)->setPage($search['page'] ? $search['page'] : 1); $board->getComments(); } catch (\Foolz\Foolslide\Model\SearchException $e) { return $this->error($e->getMessage()); } catch (\Foolz\Foolslide\Model\BoardException $e) { return $this->error($e->getMessage()); } catch (\Foolz\SphinxQL\ConnectionException $e) { return $this->error($this->preferences->get('foolslide.sphinx.custom_message', 'It appears that the search engine is offline at the moment. Please try again later.')); } // Generate the $title with all search modifiers enabled. $title = []; if ($search['text']) { array_push($title, sprintf(_i('that contain ‘%s’'), e($search['text']))); } if ($search['subject']) { array_push($title, sprintf(_i('with the subject ‘%s’'), e($search['subject']))); } if ($search['username']) { array_push($title, sprintf(_i('with the username ‘%s’'), e($search['username']))); } if ($search['tripcode']) { array_push($title, sprintf(_i('with the tripcode ‘%s’'), e($search['tripcode']))); } if ($search['filename']) { array_push($title, sprintf(_i('with the filename ‘%s’'), e($search['filename']))); } if ($search['image']) { array_push($title, sprintf(_i('with the image hash ‘%s’'), e($search['image']))); } if ($search['deleted'] == 'deleted') { array_push($title, _i('that have been deleted')); } if ($search['deleted'] == 'not-deleted') { array_push($title, _i('that has not been deleted')); } if ($search['ghost'] == 'only') { array_push($title, _i('that are by ghosts')); } if ($search['ghost'] == 'none') { array_push($title, _i('that are not by ghosts')); } if ($search['type'] == 'sticky') { array_push($title, _i('that were stickied')); } if ($search['type'] == 'op') { array_push($title, _i('that are only OP posts')); } if ($search['type'] == 'posts') { array_push($title, _i('that are only non-OP posts')); } if ($search['filter'] == 'image') { array_push($title, _i('that do not contain images')); } if ($search['filter'] == 'text') { array_push($title, _i('that only contain images')); } if ($search['capcode'] == 'user') { array_push($title, _i('that were made by users')); } if ($search['capcode'] == 'mod') { array_push($title, _i('that were made by mods')); } if ($search['capcode'] == 'admin') { array_push($title, _i('that were made by admins')); } if ($search['start']) { array_push($title, sprintf(_i('posts after %s'), e($search['start']))); } if ($search['end']) { array_push($title, sprintf(_i('posts before %s'), e($search['end']))); } if ($search['order'] == 'asc') { array_push($title, _i('in ascending order')); } if (!empty($title)) { $title = sprintf(_i('Searching for posts %s.'), implode(' ' . _i('and') . ' ', $title)); } else { $title = _i('Displaying all posts with no filters applied.'); } if ($this->radix) { $this->builder->getProps()->addTitle($title); } else { $this->builder->getProps()->addTitle('Global Search » ' . $title); } if ($board->getSearchCount() > 5000) { $search_title = sprintf(_i('%s <small>Returning only first %d of %d results found.</small>', $title, $this->preferences->get('foolslide.sphinx.max_matches', 5000), $board->getSearchCount())); } else { $search_title = sprintf(_i('%s <small>%d results found.</small>', $title, $board->getSearchCount())); } $this->param_manager->setParam('section_title', $search_title); $main_partial = $this->builder->createPartial('body', 'board'); $main_partial->getParamManager()->setParam('board', $board->getComments()); $pagination = $search; unset($pagination['page']); $pagination_arr = []; $pagination_arr[] = $this->radix !== null ? $this->radix->shortname : '_'; $pagination_arr[] = 'search'; foreach ($pagination as $key => $item) { if ($item || $item === 0) { $pagination_arr[] = rawurlencode($key); if (is_array($item)) { $item = implode('.', $item); } if ($key == 'poster_ip') { $item = Inet::dtop($item); } $pagination_arr[] = rawurlencode($item); } } $pagination_arr[] = 'page'; $this->param_manager->setParam('pagination', ['base_url' => $this->uri->create($pagination_arr), 'current_page' => $search['page'] ?: 1, 'total' => ceil($board->getCount() / 25)]); $this->param_manager->setParam('modifiers', ['post_show_board_name' => $this->radix === null, 'post_show_view_button' => true]); $this->profiler->logMem('Controller Chan $this', $this); $this->profiler->log('Controller Chan::search End'); $this->response->setCallback(function () { $this->builder->stream(); }); return $this->response; }