/** * @test * @author Christopher Hlubek <*****@*****.**> */ public function classifyDetectsLanguages() { $textcat = new Textcat(); $text = file_get_contents(__DIR__ . '/Fixtures/german.txt'); $this->assertEquals('de', $textcat->classify($text)); $text = file_get_contents(__DIR__ . '/Fixtures/english.txt'); $this->assertEquals('en', $textcat->classify($text)); $text = file_get_contents(__DIR__ . '/Fixtures/french.txt'); $this->assertEquals('fr', $textcat->classify($text)); }
/** * Fetches (new) items from the configured feed of a channel * * Adds new items and updates channels with a new last fetch date. * * @param \Planetflow3\Domain\Model\Channel $channel * @param \Closure $logCallback * @return void */ public function fetchItems(Channel $channel, $logCallback = NULL) { if ($logCallback === NULL) { $logCallback = function (Item $item, $message, $severity = LOG_INFO) { }; } $simplePie = $this->createSimplePie($channel); $existingUniversalIdentifiers = $channel->getItemsUniversalIdentifier(); $feedItems = $simplePie->get_items(); foreach ($feedItems as $feedItem) { $item = new Item(); $item->setUniversalIdentifier($feedItem->get_id()); if (isset($existingUniversalIdentifiers[$item->getUniversalIdentifier()])) { $logCallback($item, 'Skipped item, already fetched', LOG_DEBUG); continue; } $this->populateItemProperties($item, $feedItem); $this->populateItemCategories($item, $feedItem, $logCallback); if ($item->getCategories()->count() === 0 && $channel->getDefaultCategory() !== NULL) { $item->addCategory($channel->getDefaultCategory()); } if ($item->matchesChannel($channel)) { $language = $this->textcat->classify($item->getDescription() . ' ' . $item->getContent()); if ($language !== FALSE) { $item->setLanguage($language); $logCallback($item, 'Detected language ' . $language . ' for item', LOG_DEBUG); } $channel->addItem($item); $this->itemRepository->add($item); $logCallback($item, 'Item fetched and saved', LOG_INFO); $existingUniversalIdentifiers[$item->getUniversalIdentifier()] = TRUE; } else { $logCallback($item, 'Skipped item, filter not matched', LOG_DEBUG); } } $channel->setLastFetchDate(new \DateTime()); $this->channelRepository->update($channel); }