/** * starts multiple processes * * @param integer $timeout */ public function multiProcess($timeout) { if ($this->processLimit <= 1) { throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.' . PHP_EOL); } $pendingItemsStart = $this->queueRepository->countAllPendingItems(); $itemReportLimit = 20; $reportItemCount = $pendingItemsStart - $itemReportLimit; if ($this->verbose) { $this->reportItemStatus(); } $this->startRequiredProcesses(); $nextTimeOut = time() + $this->timeToLive; for ($i = 0; $i < $timeout; $i++) { $currentPendingItems = $this->queueRepository->countAllPendingItems(); if ($this->startRequiredProcesses($this->verbose)) { $nextTimeOut = time() + $this->timeToLive; } if ($currentPendingItems == 0) { if ($this->verbose) { echo 'Finished...' . chr(10); } break; } if ($currentPendingItems < $reportItemCount) { if ($this->verbose) { $this->reportItemStatus(); } $reportItemCount = $currentPendingItems - $itemReportLimit; } sleep(1); if ($nextTimeOut < time()) { $timedOutProcesses = $this->processRepository->findAll('', 'DESC', NULL, 0, 'ttl >' . $nextTimeOut); $nextTimeOut = time() + $this->timeToLive; if ($this->verbose) { echo 'Cleanup' . implode(',', $timedOutProcesses->getProcessIds()) . chr(10); } $this->crawlerObj->CLI_releaseProcesses($timedOutProcesses->getProcessIds(), true); } } if ($currentPendingItems > 0 && $this->verbose) { echo 'Stop with timeout' . chr(10); } }
/** * This method is used to show an overview about the active an the finished crawling processes * * @author Timo Schmidt * @param void * @return string */ protected function drawProcessOverviewAction() { global $BACK_PATH; $this->makeCrawlerProcessableChecks(); $crawler = $this->findCrawler(); try { $this->handleProcessOverviewActions(); } catch (Exception $e) { $this->addErrorMessage($e->getMessage()); } $offset = intval(t3lib_div::_GP('offset')); $perpage = 20; $processRepository = new tx_crawler_domain_process_repository(); $queueRepository = new tx_crawler_domain_queue_repository(); $mode = $this->pObj->MOD_SETTINGS['processListMode']; if ($mode == 'detail') { $where = ''; } elseif ($mode == 'simple') { $where = 'active = 1'; } $allProcesses = $processRepository->findAll('ttl', 'DESC', $perpage, $offset, $where); $allCount = $processRepository->countAll($where); $listView = new tx_crawler_view_process_list(); $listView->setPageId($this->pObj->id); $listView->setIconPath($BACK_PATH . '../typo3conf/ext/crawler/template/process/res/img/'); $listView->setProcessCollection($allProcesses); $listView->setCliPath($this->processManager->getCrawlerCliPath()); $listView->setIsCrawlerEnabled(!$crawler->getDisabled() && !$this->isErrorDetected); $listView->setTotalUnprocessedItemCount($queueRepository->countAllPendingItems()); $listView->setAssignedUnprocessedItemCount($queueRepository->countAllAssignedPendingItems()); $listView->setActiveProcessCount($processRepository->countActive()); $listView->setMaxActiveProcessCount(tx_crawler_api::forceIntegerInRange($this->extensionSettings['processLimit'], 1, 99, 1)); $listView->setMode($mode); $paginationView = new tx_crawler_view_pagination(); $paginationView->setCurrentOffset($offset); $paginationView->setPerPage($perpage); $paginationView->setTotalItemCount($allCount); $output = $listView->render(); if ($paginationView->getTotalPagesCount() > 1) { $output .= ' <br />' . $paginationView->render(); } return $output; }