$segmentWord = $qacMasterTableEntry[$segmentId - 1]['FORM_AR'];
$segmentWordLema = $qacMasterTableEntry[$segmentId - 1]['FEATURES']['LEM'];
$segmentWordRoot = $qacMasterTableEntry[$segmentId - 1]['FEATURES']['ROOT'];
$verseLocation = substr($location, 0, strlen($location) - 2);
//$segmentWord = removeTashkeel($segmentWord);
if ($POS == "DET") {
// second segment PoS
$segmentPoStag = $qacMasterTableEntry[$segmentId]['TAG'];
//number of segments
$numberOfSegmentsInWord = count($qacMasterTableEntry);
if (($segmentPoStag == "N" || $segmentPoStag == "ADJ") && $numberOfSegmentsInWord == 2) {
continue;
}
}
// get word index in verse
$wordIndex = getWordIndexFromQACLocation($location);
//echoN($segmentFormARimla2y);
// get simple version of the word index
$imla2yWordIndex = getImla2yWordIndexByUthmaniLocation($location);
// get verse text
$verseText = getVerseByQACLocation($QURAN_TEXT, $location);
$imla2yWord = getWordFromVerseByIndex($PAUSEMARKS, $verseText, $imla2yWordIndex);
//echoN($imla2yWord);
$stopWordsFromQuran[$imla2yWord] = 1;
echoN($imla2yWord);
}
}
$ya = "يا";
//add يا
$stopWordsFromQuran[$ya] = 1;
echoN(count($stopWordsFromQuran));
/** Returns words from QAC by PoS tags - grouped by lemma **/
function getWordsByPos(&$finalTerms, $POS)
{
global $LEMMA_TO_SIMPLE_WORD_MAP;
$qacPosEntryArr = getModelEntryFromMemory("AR", "MODEL_QAC", "QAC_POS", $POS);
$QURAN_TEXT = getModelEntryFromMemory("AR", "MODEL_CORE", "QURAN_TEXT", "");
$TOTALS = getModelEntryFromMemory("AR", "MODEL_CORE", "TOTALS", "");
$PAUSEMARKS = $TOTALS['PAUSEMARKS'];
$WORDS_FREQUENCY = getModelEntryFromMemory("AR", "MODEL_CORE", "WORDS_FREQUENCY", "");
// Get all segment in QAC for that PoS
foreach ($qacPosEntryArr as $location => $segmentId) {
$qacMasterTableEntry = getModelEntryFromMemory("AR", "MODEL_QAC", "QAC_MASTERTABLE", $location);
// get Word, Lema and root
$segmentWord = $qacMasterTableEntry[$segmentId - 1]['FORM_AR'];
$segmentWordLema = $qacMasterTableEntry[$segmentId - 1]['FEATURES']['LEM'];
$segmentWordRoot = $qacMasterTableEntry[$segmentId - 1]['FEATURES']['ROOT'];
$verseLocation = substr($location, 0, strlen($location) - 2);
//$segmentWord = removeTashkeel($segmentWord);
// get word index in verse
$wordIndex = getWordIndexFromQACLocation($location);
//$segmentFormARimla2y = $UTHMANI_TO_SIMPLE_WORD_MAP_AND_VS[$segmentWord];
// get simple version of the word index
$imla2yWordIndex = getImla2yWordIndexByUthmaniLocation($location);
// get verse text
$verseText = getVerseByQACLocation($QURAN_TEXT, $location);
//echoN("|$segmentWord|$imla2yWord");
$segmentWordNoTashkeel = removeTashkeel($segmentWordLema);
$superscriptAlef = json_decode('"\\u0670"');
$alefWasla = "ٱ";
//U+0671
//$imla2yWord = $LEMMA_TO_SIMPLE_WORD_MAP[$segmentWordLema];
// this block is important since $LEMMA_TO_SIMPLE_WORD_MAP is not good for non $superscriptAlef words
// ex زيت lemma is converted to زيتها which spoiled the ontology concept list results
if (mb_strpos($segmentWordLema, $superscriptAlef) !== false || mb_strpos($segmentWordLema, $alefWasla) !== false) {
$imla2yWord = getItemFromUthmaniToSimpleMappingTable($segmentWordLema);
if (empty($imla2yWord)) {
$imla2yWord = $LEMMA_TO_SIMPLE_WORD_MAP[$segmentWordLema];
}
} else {
$imla2yWord = getItemFromUthmaniToSimpleMappingTable($segmentWordLema);
if (empty($imla2yWord)) {
$imla2yWord = shallowUthmaniToSimpleConversion($segmentWordLema);
//$segmentWordNoTashkeel;
}
}
/// in case the word was not found after removing tashkeel, try the lema mappign table
$termWeightArr = $MODEL_CORE['WORDS_FREQUENCY']['WORDS_TFIDF'][$imla2yWord];
// NOT WORKING BECAUSE LEMMAS WILL NOT BE IN SIMPLE WORDS LIST و الصابيئن =>صَّٰبِـِٔين
// if the word after removing tashkeel is not found in quran simple words list, then try lemma table
/*if (!isset($MODEL_CORE['WORDS_FREQUENCY']['WORDS'][$imla2yWord]) )
{
$imla2yWord = $LEMMA_TO_SIMPLE_WORD_MAP[$segmentWordLema];
if ( empty($imla2yWord) )
{
echoN($segmentWordLema);
echoN($imla2yWord);
preprint_r($LEMMA_TO_SIMPLE_WORD_MAP);
preprint_r($MODEL_CORE['WORDS_FREQUENCY']['WORDS']);
exit;
}
}*/
if (empty($termWeightArr)) {
//only for weight since the lema table decrease qurana matching
$imla2yWordForWeight = $LEMMA_TO_SIMPLE_WORD_MAP[$segmentWordLema];
$termWeightArr = $WORDS_FREQUENCY['WORDS_TFIDF'][$imla2yWordForWeight];
}
$termWeight = $termWeightArr['TFIDF'];
////////////////////////////////////////////
$termWord = $segmentWordLema;
//$imla2yWord;//"|$segmentWord| ".$imla2yWord ." - $location:$segmentId - $wordIndex=$imla2yWordIndex";
if (!isset($finalTerms[$termWord])) {
$finalTerms[$termWord] = generateEmptyConceptMetadata();
$finalTerms[$termWord]['LEM'] = $segmentWordLema;
$finalTerms[$termWord]['POS'] = $POS;
$finalTerms[$termWord]['SIMPLE_WORD'] = $imla2yWord;
$finalTerms[$termWord]['ROOT'] = $segmentWordRoot;
$finalTerms[$termWord]['WEIGHT'] = $termWeight;
}
$finalTerms[$termWord]["FREQ"] = $finalTerms[$termWord]["FREQ"] + 1;
if (!isset($finalTerms[$termWord]["SEG"][$segmentWord])) {
$finalTerms[$termWord]["SEG"][$segmentWord] = $imla2yWord;
}
if (!isset($finalTerms[$termWord]["POSES"][$POS])) {
$finalTerms[$termWord]["POSES"][$POS] = 1;
}
}
return $finalTerms;
}
function getWordInfo($word, $MODEL_CORE, $MODEL_SEARCH, $MODEL_QAC, $fast = FALSE, $exactWord = FALSE)
{
$wordInfoArr = array();
$word = trim($word);
$wordUthmani = "";
$wordSimple = "";
if (isSimpleQuranWord($word)) {
$wordUthmani = getItemFromUthmaniToSimpleMappingTable($word);
$wordSimple = $word;
} else {
$wordUthmani = $word;
//preprint_r($UTHMANI_TO_SIMPLE_WORD_MAP_VS);
// tashkeel of last char is significant, ex: lemmas will probably not be in the MAP because of that
$wordSimple = getItemFromUthmaniToSimpleMappingTable($wordUthmani);
}
$WORDS_FREQUENCY = getModelEntryFromMemory("AR", "MODEL_CORE", "WORDS_FREQUENCY", "");
$freqArr = $WORDS_FREQUENCY['WORDS_TFIDF'][$wordSimple];
//preprint_r($freqArr);
$wordInfoArr['WORD_SIMPLE'] = $wordSimple;
$wordInfoArr['WORD_UTHMANI'] = $wordUthmani;
/*echoN("Simple:".$wordSimple);
echoN("Uthmani:".$wordUthmani);
echoN("Repetition:".$freqArr['TF']);
echoN("TF-IDF Weight:".round($freqArr['TFIDF'],2));
*/
$wordInfoArr['TF'] = $freqArr['TF'];
$wordInfoArr['TFIDF'] = round($freqArr['TFIDF'], 2);
//preprint_r($MODEL_QAC['QAC_MASTERTABLE']);
//preprint_r(array_keys($MODEL_QAC['QAC_FEATURES']));
$buckwalterTransliteration = "";
$posTagsArr = array();
$lemmasArr = array();
$wordRoot = "";
$featuresArr = array();
$versesArr = array();
$versesTagsArr = array();
$buckwalterTransliteration = "";
$wordRoot = "";
if (empty($wordSimple) || !modelEntryExistsInMemory("AR", "MODEL_SEARCH", "INVERTED_INDEX", $wordSimple)) {
return null;
}
//preprint_r($MODEL_SEARCH['INVERTED_INDEX'][$wordSimple]);
$invertedIndexEntry = getModelEntryFromMemory("AR", "MODEL_SEARCH", "INVERTED_INDEX", $wordSimple);
$QURAN_TEXT = getModelEntryFromMemory("AR", "MODEL_CORE", "QURAN_TEXT", "");
$TOTALS = getModelEntryFromMemory("AR", "MODEL_CORE", "TOTALS", "");
$PAUSEMARKS = $TOTALS['PAUSEMARKS'];
foreach ($invertedIndexEntry as $documentArrInIndex) {
$SURA = $documentArrInIndex['SURA'];
$AYA = $documentArrInIndex['AYA'];
$INDEX_IN_AYA_EMLA2Y = $documentArrInIndex['INDEX_IN_AYA_EMLA2Y'];
$INDEX_IN_AYA_UTHMANI = $documentArrInIndex['INDEX_IN_AYA_UTHMANI'];
$WORD_TYPE = $documentArrInIndex['WORD_TYPE'];
$EXTRA_WORD_TYPE_INFO = $documentArrInIndex['EXTRA_INFO'];
// INGORE ROOT SOURCES AND PRONOUNS, WE ONLY NEED THE NROMAL CORRESPONDING WORD
if ($WORD_TYPE == "PRONOUN_ANTECEDENT" || $WORD_TYPE == "ROOT") {
continue;
}
$qacLocation = getQACLocationStr($SURA + 1, $AYA + 1, $INDEX_IN_AYA_UTHMANI);
//echoN($qacLocation);exit;;
//preprint_r($MODEL_QAC['QAC_MASTERTABLE'][$qacLocation]);
//exit;
$qacMasterTableEntryArr = getModelEntryFromMemory("AR", "MODEL_QAC", "QAC_MASTERTABLE", $qacLocation);
// search QAC for roots and LEMMAS for this word
foreach ($qacMasterTableEntryArr as $segmentIndex => $segmentDataArr) {
$tag = $segmentDataArr['TAG'];
$segmentWord = $segmentDataArr['FORM_AR'];
//echoN($segmentWord);
//preprint_r($segmentDataArr);
$segmentWordSimple = "";
$simpleRepresentation = getItemFromUthmaniToSimpleMappingTable($segmentWord);
// !empty() produced error = Can't use function return value in write context
if (strlen(trim($simpleRepresentation)) > 0) {
$segmentWordSimple = $simpleRepresentation;
}
$buckwalterTransliteration = $segmentDataArr['FORM_EN'];
if (isset($segmentDataArr['FEATURES']['LEM'])) {
$lemma = $segmentDataArr['FEATURES']['LEM'];
}
$featuresArr = array_merge($segmentDataArr['FEATURES']);
$verseText = getVerseByQACLocation($QURAN_TEXT, $qacLocation);
$wordId = getWordIndexFromQACLocation($qacLocation);
if ($exactWord == TRUE) {
$wordFromVerseAtLocation = getWordFromVerseByIndex($PAUSEMARKS, $verseText, $wordId);
if ($wordSimple !== $wordFromVerseAtLocation) {
continue;
}
}
//echoN("$segmentWord|$tag");
//for segments like ال no corresponding simple words to compare, not our target segment, so continue
//if ( empty($segmentWordSimple)) continue;
if (isset($segmentDataArr['FEATURES']['ROOT']) && $segmentDataArr['FEATURES']['ROOT'] != -1) {
$wordRoot = $segmentDataArr['FEATURES']['ROOT'];
}
$posTagsArr[$tag] = 1;
$lemmasArr[$lemma] = 1;
//echoN("|$segmentWordSimple|$wordSimple|$segmentWord");
//$verseText = markSpecificWordInText($verseText,$wordId,$segmentWordSimple,"marked_fg");
$qacVerseLocation = substr($qacLocation, 0, strrpos($qacLocation, ":"));
if (!isset($versesArr[$qacVerseLocation])) {
$versesArr[$qacVerseLocation] = $verseText;
}
if (!isset($versesTagsArr[$qacVerseLocation])) {
$versesTagsArr[$qacVerseLocation] = "";
}
$versesTagsArr[$qacVerseLocation] = $versesTagsArr[$qacVerseLocation] . " " . $tag;
}
// we don't need all inverted index list except for verses, only break if we found at least one word
if ($fast == true && !empty($versesArr)) {
break;
}
}
$wordInfoArr['BUCKWALTER'] = $buckwalterTransliteration;
$wordInfoArr['ROOT'] = $wordRoot;
$wordInfoArr['LEM'] = $lemmasArr;
$wordInfoArr['POS'] = $posTagsArr;
$wordInfoArr['VERSES'] = $versesArr;
$wordInfoArr['VERSES_POS_TAGS'] = $versesTagsArr;
$wordInfoArr['FEATURES'] = $featuresArr;
return $wordInfoArr;
}
foreach ($featuresArr as $oneFeature) {
$isFeatureFound = isset($qacMasterTableEntry[$segmentId - 1]['FEATURES'][$features]);
}
if ($isFeatureFound == false) {
continue;
}
}
if (!isset($unrepeatedWords[$segmentWord])) {
$unrepeatedWords[$segmentWord] = 1;
}
if (!isset($markedVerses[$verseLocation])) {
$verseText = getVerseByQACLocation($QURAN_TEXT, $location);
} else {
$verseText = $markedVerses[$verseLocation];
}
$wordId = getWordIndexFromQACLocation($location) - 1;
$verseText = markSpecificWordInText($verseText, $wordId, $segmentWord, "marked_fg");
$markedVerses[$verseLocation] = $verseText;
}
?>
<div id='pos-words-verses-statistics'>
<b><?php
echo addCommasToNumber($allOccurencesCount);
?>
</b> All Segments -
<b><?php
echo addCommasToNumber(count($unrepeatedWords));
?>
</b> Distinct Words -
