/**
* Checks the density of links within a node, is there not much text and most of it contains linky shit?
* if so it's no good
*
* @param Element $node
* @param double $limit
*
* @return bool
*/
private function isHighLinkDensity(Element $node, $limit = 1.0)
{
$links = $node->find('a, [onclick]');
if ($links->count() == 0) {
return false;
}
$words = preg_split('@[\\s]+@iu', $node->text(), -1, PREG_SPLIT_NO_EMPTY);
if (count($words) == 0) {
return false;
}
$sb = [];
foreach ($links as $link) {
$sb[] = Helper::textNormalise($link->text());
}
$linkText = implode('', $sb);
$linkWords = explode(' ', $linkText);
$numberOfLinkWords = count($linkWords);
$numberOfLinks = $links->count();
$linkDivisor = $numberOfLinkWords / count($words);
$score = $linkDivisor * $numberOfLinks;
if ($score >= $limit) {
return true;
}
return false;
}
/**
* @param Element $node
*
* @return LocallyStoredImage[]
*/
private function getImageCandidates(Element $node)
{
$images = $node->find('img');
$filteredImages = $this->filterBadNames($images);
$goodImages = $this->findImagesThatPassByteSizeTest($filteredImages);
return $goodImages;
}
/**
* Stores how many decent nodes are under a parent node
*
* @param Element $node
* @param int $addToCount
*/
private function updateNodeCount(Element $node, $addToCount)
{
$currentScore = (int) $node->attr('gravityNodes');
$node->attr('gravityNodes', $currentScore + $addToCount);
}
/**
* we could have long articles that have tons of paragraphs so if we tried to calculate the base score against
* the total text score of those paragraphs it would be unfair. So we need to normalize the score based on the average scoring
* of the paragraphs within the top node. For example if our total score of 10 paragraphs was 1000 but each had an average value of
* 100 then 100 should be our base.
*
* @param Element $topNode
*
* @return int
*/
private function getBaselineScoreForSiblings(Element $topNode)
{
$base = 100000;
$numberOfParagraphs = 0;
$scoreOfParagraphs = 0;
$nodesToCheck = $topNode->find('p, strong');
foreach ($nodesToCheck as $node) {
$nodeText = $node->text();
$wordStats = $this->config()->getStopWords()->getStopwordCount($nodeText);
$highLinkDensity = $this->isHighLinkDensity($node);
if ($wordStats->getStopWordCount() > 2 && !$highLinkDensity) {
$numberOfParagraphs += 1;
$scoreOfParagraphs += $wordStats->getStopWordCount();
}
}
if ($numberOfParagraphs > 0) {
$base = $scoreOfParagraphs / $numberOfParagraphs;
}
return $base;
}
/**
* Generate <p> element replacements for supplied elements child nodes as required.
*
* @param Element $node
*
* @return \DOMWrap\NodeList $nodesToReturn Replacement elements
*/
private function getReplacementNodes(Element $node)
{
$nodesToReturn = $node->newNodeList();
$nodesToRemove = $node->newNodeList();
$replacementNodes = $node->newNodeList();
$fnCompareSiblingNodes = function ($node) {
if ($node->is(':not(a)') || $node->nodeType == XML_TEXT_NODE) {
return true;
}
};
foreach ($node->contents() as $child) {
if ($child->is('p') && $replacementNodes->count()) {
$nodesToReturn[] = $this->getFlushedBuffer($replacementNodes);
$replacementNodes->fromArray([]);
$nodesToReturn[] = $child;
} else {
if ($child->nodeType == XML_TEXT_NODE) {
$replaceText = $child->text();
if (!empty($replaceText)) {
// Get all previous sibling <a> nodes, the current text node, and all next sibling <a> nodes.
$siblings = $child->precedingUntil($fnCompareSiblingNodes, 'a')->merge([$child])->merge($child->followingUntil($fnCompareSiblingNodes, 'a'));
foreach ($siblings as $sibling) {
// Place current nodes textual contents in-between previous and next nodes.
if ($sibling->isSameNode($child)) {
$replacementNodes[] = new Text($replaceText);
// Grab the contents of any unprocessed <a> siblings and flag them for removal.
} else {
if ($sibling->getAttribute('grv-usedalready') != 'yes') {
$sibling->setAttribute('grv-usedalready', 'yes');
$replacementNodes[] = $sibling->cloneNode(true);
$nodesToRemove[] = $sibling;
}
}
}
}
$nodesToRemove[] = $child;
} else {
if ($replacementNodes->count()) {
$nodesToReturn[] = $this->getFlushedBuffer($replacementNodes);
$replacementNodes->fromArray([]);
}
$nodesToReturn[] = $child;
}
}
}
// Flush any remaining replacementNodes left over from text nodes.
if ($replacementNodes->count()) {
$nodesToReturn[] = $this->getFlushedBuffer($replacementNodes);
}
// Remove potential duplicate <a> tags.
foreach ($nodesToReturn as $key => $return) {
if ($nodesToRemove->exists($return)) {
unset($nodesToReturn[$key]);
}
}
$nodesToRemove->remove();
return $nodesToReturn;
}
/**
* A lot of times the first paragraph might be the caption under an image so we'll want to make sure if we're going to
* boost a parent node that it should be connected to other paragraphs, at least for the first n paragraphs
* so we'll want to make sure that the next sibling is a paragraph and has at least some substantial weight to it
*
* @param Element $node
*
* @return bool
*/
private function isOkToBoost(Element $node)
{
$stepsAway = 0;
$minimumStopWordCount = 5;
$maxStepsAwayFromNode = 3;
// Find all previous sibling element nodes
$siblings = $node->precedingAll(function ($node) {
return $node instanceof Element;
});
foreach ($siblings as $sibling) {
if ($sibling->is('p, strong')) {
if ($stepsAway >= $maxStepsAwayFromNode) {
return false;
}
$wordStats = $this->config()->getStopWords()->getStopwordCount($sibling->text());
if ($wordStats->getStopWordCount() > $minimumStopWordCount) {
return true;
}
$stepsAway += 1;
}
}
return false;
}