/**
* Makes an n or all word gram Bloom filter based on the supplied arguments
* Wikipedia files are assumed to have been place in the PREP_DIR before this
* is run and writes it into the resources folder of the given locale
*
* @param array $args command line arguments with first two elements of $argv
* removed. For details on which arguments do what see the $usage variable
*/
function makeNWordGramsFiles($args)
{
if (!isset($args[1])) {
$args[1] = "en";
$args[2] = "en-US";
}
if (!isset($args[2])) {
$args[2] = $args[1];
}
if (!isset($args[3])) {
$args[3] = 2;
// bigrams
}
if (!isset($argv[4])) {
$args[4] = NWordGrams::PAGE_COUNT_WIKIPEDIA;
}
if (!isset($args[5]) && $args[3] == "all" && $args[2] == NWordGrams::PAGE_COUNT_WIKIPEDIA) {
$args[5] = 400000;
} else {
$args[5] = -1;
}
$wiki_file_path = PREP_DIR . "/";
if (!file_exists($wiki_file_path . $args[0])) {
echo $args[0] . " does not exist in {$wiki_file_path}";
exit;
}
/*
*This call creates a ngrams text file from input xml file and
*returns the count of ngrams in the text file.
*/
list($num_ngrams, $max_gram_len) = NWordGrams::makeNWordGramsTextFile($args[0], $args[1], $args[2], $args[3], $args[4], $args[5]);
/*
*This call creates a bloom filter file from n word grams text file based
*on the language specified.The lang passed as parameter is prefixed
*to the filter file name. The count of n word grams in text file is passed
*as a parameter to set the limit of n word grams in the filter file.
*/
NWordGrams::makeNWordGramsFilterFile($args[2], $args[3], $num_ngrams, $max_gram_len);
}
