public function execute($debug)
{
$startTime = microtime(true);
foreach (Kwf_Util_Fulltext_Backend_Abstract::getInstance()->getSubroots() as $subroot) {
$t = time();
if ($debug) {
echo "\n[{$subroot}] check-for-invalid...\n";
}
$cmd = Kwf_Config::getValue('server.phpCli') . " bootstrap.php fulltext check-for-invalid-subroot --subroot={$subroot}";
if ($debug) {
$cmd .= " --debug";
}
//if ($this->_getParam('silent')) $cmd .= " --silent";
passthru($cmd, $ret);
if ($ret) {
exit($ret);
}
if ($debug) {
echo "[{$subroot}] check-for-invalid finished: " . Kwf_View_Helper_SecondsAsDuration::secondsAsDuration(time() - $t) . "\n\n";
}
$t = time();
if ($debug) {
echo "\n[{$subroot}] check-contents...\n";
}
$cmd = Kwf_Config::getValue('server.phpCli') . " bootstrap.php fulltext check-contents-subroot --subroot={$subroot}";
if (isset($this->_options['skipDiff'])) {
$cmd .= " --skip-diff";
}
if ($debug) {
$cmd .= " --debug";
}
//if ($this->_getParam('silent')) $cmd .= " --silent";
passthru($cmd, $ret);
if ($ret) {
exit($ret);
}
if ($debug) {
echo "[{$subroot}] check-contents finished: " . Kwf_View_Helper_SecondsAsDuration::secondsAsDuration(time() - $t) . "\n\n";
}
$t = time();
if ($debug) {
echo "\n[{$subroot}] optimize...\n";
}
Kwf_Util_Fulltext_Backend_Abstract::getInstance()->optimize($debug);
if ($debug) {
echo "[{$subroot}] optimize finished: " . Kwf_View_Helper_SecondsAsDuration::secondsAsDuration(time() - $t) . "\n\n";
}
}
if ($debug) {
echo "\ncomplete fulltext check-contents finished: " . Kwf_View_Helper_SecondsAsDuration::secondsAsDuration(microtime(true) - $startTime) . "s\n";
}
}
private function _query($select)
{
$limitCount = $select->getPart(Kwf_Model_Select::LIMIT_COUNT);
$limitOffset = $select->getPart(Kwf_Model_Select::LIMIT_OFFSET);
if ($limitOffset === null) {
$limitOffset = 0;
}
$queryString = '';
$params = array();
if ($select->getPart(Kwf_Model_Select::WHERE_EXPRESSION)) {
foreach ($select->getPart(Kwf_Model_Select::WHERE_EXPRESSION) as $exp) {
if ($exp instanceof Kwf_Model_Select_Expr_SearchLike) {
foreach ($exp->getSearchValues() as $field => $value) {
if ($field == 'query') {
$queryString = $value;
}
}
} else {
$query = $this->_getSearchQueryByExpr($exp);
if (isset($params['fq'])) {
$params['fq'] .= '+' . $query;
} else {
$params['fq'] = $query;
}
}
}
}
if ($select->getPart(Kwf_Model_Select::WHERE_NOT_EQUALS)) {
foreach ($select->getPart(Kwf_Model_Select::WHERE_NOT_EQUALS) as $field => $value) {
if (isset($params['fq'])) {
//if more than one fields should be queried they are apended with +
$params['fq'] .= '+' . '-' . $field . ':' . $value;
} else {
//format is: -field:value for negation
$params['fq'] = '-' . $field . ':' . $value;
}
}
}
if ($id = $select->getPart(Kwf_Model_Select::WHERE_ID)) {
throw new Kwf_Exception_NotYetImplemented();
}
$subroot = Kwf_Component_Data_Root::getInstance();
if ($select->getPart(Kwf_Model_Select::WHERE_EQUALS)) {
foreach ($select->getPart(Kwf_Model_Select::WHERE_EQUALS) as $field => $value) {
if ($field == 'subroot') {
$subroot = $value;
}
}
}
$res = Kwf_Util_Fulltext_Backend_Abstract::getInstance()->userSearch($subroot, $queryString, $limitOffset, $limitCount, $params);
if ($res['error']) {
throw new Kwf_Exception("Fulltext error: " . $res['error']);
}
return $res;
}
public function checkPagesSubrootAction()
{
Kwf_Util_MemoryLimit::set(256);
$pageClassesThatCanHaveFulltext = array();
foreach (self::_getAllPossiblePageComponentClasses() as $c) {
if (self::_canHaveFulltext($c)) {
$pageClassesThatCanHaveFulltext[] = $c;
}
}
$startTime = microtime(true);
if (!$this->_getParam('componentId')) {
throw new Kwf_Exception_Client("componentId parameter required");
}
$componentId = $this->_getParam('componentId');
$queue = array($componentId);
$stats = array('pages' => 0, 'indexedPages' => 0);
while ($queue) {
if (!$queue) {
break;
}
if ($this->_getParam('debug')) {
echo "queued: " . count($queue) . ' :: ' . round(memory_get_usage() / 1024 / 1024, 2) . "MB\n";
}
$componentId = array_shift($queue);
$stats['pages']++;
//if ($this->_getParam('debug')) echo "==> ".$componentId;
$page = Kwf_Component_Data_Root::getInstance()->getComponentById($componentId);
//if ($this->_getParam('debug')) echo " :: $page->url\n";
if (!$page) {
if ($this->_getParam('debug')) {
echo "{$componentId} not found!\n";
}
continue;
}
//echo "$page->url\n";
if ($this->_getParam('verbose')) {
echo "getting child pages...";
}
$childPages = $page->getChildPseudoPages(array('pageGenerator' => false, 'componentClasses' => $pageClassesThatCanHaveFulltext), array('pseudoPage' => false));
$childPages = array_merge($childPages, $page->getChildPseudoPages(array('pageGenerator' => true), array('pseudoPage' => false)));
if ($this->_getParam('verbose')) {
echo " done\n";
}
foreach ($childPages as $c) {
$i = $c;
do {
if (Kwc_Abstract::getFlag($i->componentClass, 'skipFulltextRecursive')) {
continue 2;
}
} while ($i = $i->parent);
if ($this->_getParam('verbose')) {
echo "queued {$c->componentId}\n";
}
$queue[] = $c->componentId;
}
unset($c);
$hasFulltext = false;
if (!Kwc_Abstract::getFlag($page->componentClass, 'skipFulltext') && $page->isPage && ($page->getRecursiveChildComponents(array('flag' => 'hasFulltext', 'inherit' => false)) || Kwc_Abstract::getFlag($page->componentClass, 'hasFulltext'))) {
$hasFulltext = true;
}
if ($hasFulltext) {
if (!Kwf_Util_Fulltext_Backend_Abstract::getInstance()->documentExists($page)) {
if (Kwf_Util_Fulltext_Backend_Abstract::getInstance()->indexPage($page, $this->_getParam('debug'))) {
$stats['indexedPages']++;
if (!$this->_getParam('silent')) {
echo "not found in index: {$page->componentId} has content!!!!\n";
}
} else {
if ($this->_getParam('debug')) {
echo "not found in index: {$page->componentId} has NO content (that's ok)\n";
}
}
}
}
unset($page);
if ($stats['pages'] % 50 == 0) {
Kwf_Component_Data_Root::getInstance()->freeMemory();
}
}
if (!$this->_getParam('silent')) {
echo "processed pages: {$stats['pages']}\n";
echo "indexed pages: {$stats['indexedPages']}\n";
}
exit;
}
public function checkContentsSubrootAction()
{
Kwf_Util_MemoryLimit::set(256);
$subroot = Kwf_Component_Data_Root::getInstance()->getComponentById($this->_getParam('subroot'));
if (!$subroot) {
$subroot = Kwf_Component_Data_Root::getInstance();
}
$i = 0;
$stats = array('indexedPages' => 0, 'diffPages' => 0);
$pagesMetaModel = Kwf_Component_PagesMetaModel::getInstance();
$select = new Kwf_Model_Select();
$select->whereEquals('deleted', false);
$select->whereEquals('fulltext_skip', false);
$select->whereEquals('subroot_component_id', $subroot->componentId);
$it = new Kwf_Model_Iterator_Packages(new Kwf_Model_Iterator_Rows($pagesMetaModel, $select));
if ($this->_getParam('debug')) {
$it = new Kwf_Iterator_ConsoleProgressBar($it);
}
foreach ($it as $row) {
$componentId = $row->page_id;
$page = Kwf_Component_Data_Root::getInstance()->getComponentById($componentId);
if (!$this->_getParam('skip-diff')) {
$docContent = Kwf_Util_Fulltext_Backend_Abstract::getInstance()->getDocumentContent($page);
} else {
$docContent = '';
}
$newDoc = Kwf_Util_Fulltext_Backend_Abstract::getInstance()->getFulltextContentForPage($page);
if (!$newDoc) {
//this can happen (if there is no content)
Kwf_Util_Fulltext_Backend_Abstract::getInstance()->deleteDocument($subroot, $componentId);
$row = $pagesMetaModel->getRow($componentId);
if ($row) {
$row->delete();
}
continue;
}
if (trim($newDoc['content']) != trim($docContent)) {
$stats['diffPages']++;
if (Kwf_Util_Fulltext_Backend_Abstract::getInstance()->indexPage($page)) {
$stats['indexedPages']++;
}
if (!$this->_getParam('silent') && !$this->_getParam('skip-diff')) {
echo "DIFF: {$componentId}\n";
}
}
unset($page);
if ($i++ % 10) {
Kwf_Component_Data_Root::getInstance()->freeMemory();
}
//if ($this->_getParam('debug')) echo "memory_usage ".(memory_get_usage()/(1024*1024))."MB\n";
}
if ($this->_getParam('debug')) {
if (!$this->_getParam('skip-diff')) {
echo "pages with diff: {$stats['diffPages']}\n";
}
echo "indexed pages: {$stats['indexedPages']}\n";
}
exit;
}