function PatternMatcherOnig($t_args, $inputs)
{
grokit_assert(\count($inputs) == 1, 'PatternMatcherOnig GF only supports 1 input!');
$pattern = get_first_key($t_args, ['pattern']);
$inName = array_keys($inputs)[0];
$inType = array_get_index($inputs, 0);
$inTypeString = $inType->name();
$validTypes = ['BASE::STRING_LITERAL'];
grokit_assert(in_array($inTypeString, $validTypes), 'Unsupported input type ' . $inTypeString);
$className = generate_name('PatternMatcherOnigGF');
?>
class <?php
echo $className;
?>
{
PatternMatcherOnig matcher;
public:
<?php
echo $className;
?>
() :
matcher("<?php
echo $pattern;
?>
")
{ }
bool Filter( const <?php
echo $inType;
?>
& <?php
echo $inName;
?>
) {
return matcher.Match(<?php
echo $inName;
?>
);
}
};
<?php
return ['kind' => 'GF', 'name' => $className, 'input' => $inputs, 'user_headers' => ['PatternMatcherOnig.h']];
}
/**
* GI that generates data in clusters, using a specified distribution for each
* cluster.
*
* This GI requires the following template arguments:
* - 'n' or 0
* The number of tuples to generate. Note: this value is per task.
* The total number of tuples generated will be n_tasks * n
* - 'centers' or 1
* A list of configuration for the centers.
*
* The following template arguments are optional:
* - 'outputs'
* If the outputs of the GI are not given implicitly, they can be
* specified in this template argument. The number of dimensions will
* be determined by the number of outputs.
*
* All output types must be numeric real types. The default type for
* outputs is DOUBLE.
* - 'dist.lib' = 'std'
* Which library to use for generating distributions.
* Valid options are:
* - std
* - boost
* - 'seed' = null
* The seed to be used for the random number generator. This seed will
* be used to generate the seed for each task, and different runs with
* the same seed will produce the same data.
* - 'compute.sets' = 1
* The number of sets of tuples to compute at once.
*
* Each center configuration is a functor with the form:
* dist_name(args)
*
* The following distributions are supported:
* { Uniform Distributions }
* - uniform(a = 0, b = 1)
*
* { Normal Distributions }
* - normal(mean = 0.0, std_dev = 1.0) [ synonyms: gaussian ]
* - inverse_gaussian(mean = 1, shape = 1) [ synonyms: inverse_normal ]
*
* { Bernoulli Distributions }
* - binomial(t = 1, p = 0.5)
* - negative_binomial(k = 1, p = 0.5)
*
* { Poisson Distributions }
* - exponential( lambda = 1 )
* - gamma(alpha = 1, beta = 1) [ synonyms: Gamma ]
*/
function ClusterGen(array $t_args, array $outputs)
{
$sys_headers = ['array', 'cinttypes'];
$user_headers = [];
$libraries = [];
if (\count($outputs) == 0) {
grokit_assert(array_key_exists('outputs', $t_args), 'ClusterGen: No outputs specified');
$count = 0;
foreach ($t_args['outputs'] as $type) {
if (is_identifier($type)) {
$type = lookupType($type);
}
grokit_assert(is_datatype($type), 'ClusterGen: Non data-type ' . $type . ' given as output');
$name = 'output' . $count++;
$outputs[$name] = $type;
}
}
foreach ($outputs as $name => &$type) {
if (is_null($type)) {
$type = lookupType('base::DOUBLE');
} else {
grokit_assert($type->is('real'), 'ClusterGen: Non-real datatype ' . $type . ' given as output');
}
}
$myOutputs = [];
foreach ($outputs as $name => $type) {
$myOutputs[$name] = $type;
}
$tSize = \count($outputs);
$seed = get_default($t_args, 'seed', null);
if ($seed !== null) {
grokit_assert(is_int($seed), 'ClusterGen: Seed must be an integer or null.');
} else {
$user_headers[] = 'HashFunctions.h';
}
$distLib = get_default($t_args, 'dist.lib', 'std');
$distNS = '';
switch ($distLib) {
case 'std':
$sys_headers[] = 'random';
$distNS = 'std';
break;
case 'boost':
$sys_headers[] = 'boost/random.hpp';
$distNS = 'boost::random';
$libraries[] = 'boost_random-mt';
if ($seed === null) {
// Need random_device
$sys_headers[] = 'boost/random/random_device.hpp';
$libraries[] = 'boost_system-mt';
}
break;
default:
grokit_error('ClusterGen: Unknown RNG library ' . $distLib);
}
$distRNG = 'mt19937';
$RNGtype = $distNS . '::' . $distRNG;
$nTuples = get_first_key($t_args, ['n', '0']);
grokit_assert(is_int($nTuples), 'ClusterGen: the number of tuples to be produced must be an integer.');
$centers = get_first_key($t_args, ['centers', 1]);
grokit_assert(is_array($centers), 'ClusterGen: centers must be an array of functors');
$handleDist = function ($name, $args, $oType) use($distNS) {
$distName = '';
$distArgs = [];
switch ($name) {
case 'gaussian':
case 'normal':
$distName = $distNS . '::' . 'normal_distribution<' . $oType . '>';
grokit_assert(\count($args) <= 2, 'ClusterGen: Normal distribution takes at most 2 arguments, ' . \count($args) . ' given');
$mean = get_default($args, ['mean', 0], 0.0);
$sigma = get_default($args, ['std_dev', 'sigma', 1], 1.0);
grokit_assert(is_numeric($mean), 'ClusterGen: mean parameter of binomial distribution must be a real number.');
grokit_assert(is_numeric($sigma), 'ClusterGen: sigma parameter of binomial distribution must be a real number.');
$mean = floatval($mean);
$sigma = floatval($sigma);
$distArgs = [$mean, $sigma];
break;
case 'binomial':
$distName = $distNS . '::' . 'binomial_distribution<' . $oType . '>';
grokit_assert(\count($args) <= 2, 'ClusterGen: Binomial distribution takes at most 2 arguments, ' . \count($args) . ' given');
$t = get_default($args, ['t', 0], 1);
$p = get_default($args, ['p', 1], 0.5);
grokit_assert(is_int($t), 'ClusterGen: t parameter of binomial distribution must be an integer.');
grokit_assert(is_numeric($p), 'ClusterGen: p parameter of binomial distribution must be a real number.');
$p = floatval($p);
grokit_assert($p >= 0 && $p <= 1, 'ClusterGen: p parameter of binomial distribution must be in the range [0, 1]');
grokit_assert($t >= 0, 'ClusterGen: t parameter of binomial distribution must be in the range [0, +inf)');
$distArgs = [$t, $p];
break;
case 'negative_binomial':
$distName = $distNS . '::' . 'negative_binomial_distribution<' . $oType . '>';
grokit_assert(\count($args) <= 2, 'ClusterGen: Negative Binomial distribution takes at most 2 arguments, ' . \count($args) . ' given');
$k = get_default($args, ['k', 0], 1);
$p = get_default($args, ['p', 1], 0.5);
grokit_assert(is_int($k), 'ClusterGen: k parameter of binomial distribution must be an integer.');
grokit_assert(is_numeric($p), 'ClusterGen: p parameter of binomial distribution must be a real number.');
$p = floatval($p);
grokit_assert($p > 0 && $p <= 1, 'ClusterGen: p parameter of negative binomial distribution must be in the range (0, 1]');
grokit_assert($k > 0, 'ClusterGen: k parameter of negative binomial distribution must be in the range (0, +inf)');
$distArgs = [$k, $p];
break;
case 'inverse_gaussian':
case 'inverse_normal':
grokit_assert(\count($args) <= 2, 'ClusterGen: Inverse Gaussian distribution takes at most 2 arguments, ' . \count($args) . ' given');
$mean = get_default($args, ['mean', 0], 1);
$shape = get_default($args, ['shape', 1], 1);
grokit_assert(is_numeric($mean), 'ClusterGen: mean parameter of inverse gaussian distribution must be a real number.');
grokit_assert(is_numeric($shape), 'ClusterGen: shape parameter of inverse gaussian distribution must be a real number.');
$mean = floatval($mean);
$shape = floatval($shape);
grokit_assert($mean > 0, 'ClusterGen: mean of inverse gaussian distribution must be in range (0, inf)');
grokit_assert($shape > 0, 'ClusterGen: shape of inverse gaussian distribution must be in range (0, inf)');
$gen_args = ['output' => $oType, 'ns' => $distNS];
$distName = strval(lookupResource('datagen::InverseGaussianGen', $gen_args));
$distArgs = [$mean, $shape];
break;
case 'uniform':
$distName = $distNS . '::' . 'uniform_real_distribution<' . $oType . '>';
grokit_assert(\count($args) <= 2, 'ClusterGen: Uniform distribution takes at most 2 arguments, ' . \count($args) . ' given');
$a = get_default($args, ['a', 0], 0.0);
$b = get_default($args, ['b', 1], 1.0);
grokit_assert(is_numeric($a), 'ClusterGen: `a` parameter of uniform distribution must be a real number.');
grokit_assert(is_numeric($b), 'ClusterGen: `b` parameter of uniform distribution must be a real number.');
$a = floatval($a);
$b = floatval($b);
grokit_assert($b >= $a, 'ClusterGen: `b` parameter of uniform distribution must be >= the `a` parameter.');
$distArgs = [$a, $b];
break;
case 'exponential':
$distName = $distNS . '::' . 'exponential_distribution<' . $oType . '>';
grokit_assert(\count($args) <= 1, 'ClusterGen: Exponential distribution takes at most 1 argument.');
$lambda = get_default($args, ['lambda', 0], 1.0);
grokit_assert(is_numeric($lambda), 'ClusterGen: `lambda` parameter of exponential distribution must be a real number.');
$lambda = floatval($lambda);
grokit_assert($lambda > 0, 'ClusterGen: `lambda` parameter of exponential distribution must be in range (0, +inf).');
$distArgs = [$lambda];
break;
case 'gamma':
case 'Gamma':
$distName = $distNS . '::' . 'gamma_distribution<' . $oType . '>';
grokit_assert(\count($args) <= 2, 'ClusterGen: Gamma distribution takes at most 2 arguments.');
$alpha = get_default($args, ['alpha', 0], 1.0);
$beta = det_default($args, ['beta', 1], 1.0);
grokit_assert(is_numeric($alpha), 'ClusterGen: `alpha` parameter of gamma distribution must be a real number.');
grokit_assert(is_numeric($beta), 'ClusterGen: `beta` parameter of gamma distribution must be a real number.');
$alpha = floatval($alpha);
$beta = floatval($beta);
$distArgs = [$alpha, $beta];
break;
default:
grokit_error('ClusterGen: Unknown distribution ' . $name . ' given for center');
}
return [$distName, $distArgs];
};
$dists = [];
$distArgs = [];
$count = 0;
$oType = '';
$nCenters = 1;
reset($outputs);
foreach ($centers as $val) {
$cluster = $val;
if (is_functor($val)) {
$cluster = [$val];
} else {
if (is_array($val)) {
$nCenters = lcm($nCenters, \count($val));
} else {
grokit_error('ClusterGen: center descriptions must be functors or list of functors');
}
}
$curDist = [];
$curDistArgs = [];
$curDistName = 'distribution' . $count++;
$oType = strval(current($outputs));
$iCount = 0;
foreach ($cluster as $functor) {
grokit_assert(is_functor($functor), 'ClusterGen: center description must be a functor');
$vName = $curDistName . '_' . $iCount++;
$ret = $handleDist($functor->name(), $functor->args(), $oType);
$curDist[$vName] = $ret[0];
$curDistArgs[$vName] = $ret[1];
}
next($outputs);
$dists[$curDistName] = $curDist;
$distArgs[$curDistName] = $curDistArgs;
}
// Determine the default number of sets to compute at a time.
// We want to generate either $nTuples or 10,000 tuples, depending on which
// is less.
$defaultSetsTarget = min($nTuples, 10000);
$setsToTarget = intval(ceil($defaultSetsTarget / $nCenters));
$computeSets = get_default($t_args, 'compute.sets', $setsToTarget);
grokit_assert(is_int($computeSets) && $computeSets > 0, 'ClusterGen: compute.sets must be a positive integer, ' . $computeSets . ' given');
$className = generate_name('ClusterGen');
// For some BIZZARE reason, the $outputs array was getting modified while
// traversing over the $dists array. Making a deep copy of the outputs and
// then reassigning it seems to fix the issue.
$outputs = $myOutputs;
?>
class <?php
echo $className;
?>
{
// The number of tuples to produce per task
static constexpr size_t N = <?php
echo $nTuples;
?>
;
static constexpr size_t CacheSize = <?php
echo $computeSets * $nCenters;
?>
;
// Typedefs
typedef std::tuple<<?php
echo array_template('{val}', ', ', $outputs);
?>
> Tuple;
typedef std::array<Tuple, CacheSize> TupleArray;
typedef TupleArray::const_iterator TupleIterator;
typedef <?php
echo $RNGtype;
?>
RandGen;
// Number of tuples produced.
uintmax_t count;
// Cache a number of outputs for efficiency
TupleArray cache;
TupleIterator cacheIt;
// Random number generator
RandGen rng;
// Distributions
<?php
// This is the section causing issues.
foreach ($dists as $name => $list) {
foreach ($list as $vName => $type) {
?>
<?php
echo $type;
?>
<?php
echo $vName;
?>
;
<?php
}
// foreach distribution
}
// foreach cluster set
?>
// Helper function to generate tuples.
void GenerateTuples(void) {
<?php
$tIndex = 0;
foreach ($dists as $name => $list) {
$lCenters = \count($list);
// $nCenters has been defined to be the LCM of the number of centers in
// any column, so $lCenter is guaranteed to divide evenly into
// CacheSize
?>
for( size_t index = 0; CacheSize > index; index += <?php
echo $lCenters;
?>
) {
<?php
$index = 0;
foreach ($list as $vName => $type) {
?>
std::get<<?php
echo $tIndex;
?>
>(cache[index + <?php
echo $index;
?>
]) = <?php
echo $vName;
?>
(rng);
<?php
$index++;
}
// foreach value in tuple
?>
}
<?php
$tIndex++;
}
// foreach distribution
?>
cacheIt = cache.cbegin();
}
public:
// Constructor
<?php
echo $className;
?>
( GIStreamProxy & _stream ) :
cache()
, cacheIt()
, count(0)
, rng()
<?php
foreach ($dists as $name => $list) {
foreach ($list as $vName => $type) {
?>
, <?php
echo $vName;
?>
(<?php
echo implode(', ', $distArgs[$name][$vName]);
?>
)
<?php
}
// foreach distribution
}
// foreach cluster set
?>
{
<?php
if (is_null($seed)) {
?>
<?php
echo $distNS;
?>
::random_device rd;
<?php
}
// if seed is null
?>
RandGen::result_type seed = <?php
echo is_null($seed) ? 'rd()' : "CongruentHash({$seed}, _stream.get_id() )";
?>
;
rng.seed(seed);
cacheIt = cache.cend();
}
// Destructor
~<?php
echo $className;
?>
(void) { }
bool ProduceTuple(<?php
echo typed_ref_args($outputs);
?>
) {
if( N > count ) {
if( cacheIt == cache.cend() ) {
GenerateTuples();
}
<?php
$tIndex = 0;
foreach ($outputs as $name => $type) {
?>
<?php
echo $name;
?>
= std::get<<?php
echo $tIndex;
?>
>(*cacheIt);
<?php
$tIndex++;
}
// foreach output
?>
++cacheIt;
++count;
return true;
}
else {
return false;
}
}
};
<?php
return array('kind' => 'GI', 'name' => $className, 'output' => $outputs, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'libraries' => $libraries);
}
function ExtremeTuples(array $t_args, array $inputs, array $outputs)
{
$extremes = get_first_key($t_args, ['extremes']);
$nExt = \count($extremes);
grokit_assert($nExt > 0, 'No extremes specified for ExtremeTuples GLA.');
if (\count($inputs) == 0) {
grokit_assert(array_key_exists('inputs', $t_args), 'No arguments specified for ExtremeTuples GLA.');
$count = 0;
foreach ($t_args['inputs'] as $type) {
if (is_identifier($type)) {
$type = lookupType(strval($type));
}
grokit_assert(is_datatype($type), 'Only datatypes can be specified as inputs to ' . 'the ExtremeTuples GLA');
$name = 'et_val' . $count;
$inputs[$name] = $type;
}
}
$outputMap = [];
reset($outputs);
foreach ($inputs as $name => $type) {
$oKey = key($outputs);
$outputs[$oKey] = $type;
$outputMap[$oKey] = $name;
next($outputs);
}
grokit_assert($nExt <= \count($inputs), 'There can not be more extreme values than there are inputs!');
$mainAtts = [];
$extraAtts = [];
$minOpts = ['MIN', 'MINIMUM', '-', '<'];
$maxOpts = ['MAX', 'MAXIMUM', '+', '>'];
$inArrayCase = function ($needle, $haystack) {
foreach ($haystack as $item) {
if (strcasecmp($needle, $item) == 0) {
return true;
}
}
return false;
};
$minimum = [];
foreach ($extremes as $name => $val) {
grokit_assert(array_key_exists($name, $inputs), "ExtremeTuples: Expression with name " . $name . " specified as extreme not found in inputs");
}
foreach ($inputs as $name => $type) {
if (array_key_exists($name, $extremes)) {
$mainAtts[$name] = $type;
if ($inArrayCase($extremes[$name], $minOpts)) {
$minimum[$name] = true;
} else {
if ($inArrayCase($extremes[$name], $maxOpts)) {
$minimum[$name] = false;
} else {
grokit_error('Unknown extreme type ' . $extremes[$name] . ' specified for ' . $name);
}
}
} else {
$extraAtts[$name] = $type;
}
}
$debug = get_default($t_args, 'debug', 0);
$className = generate_name('ExtremeTuples');
?>
class <?php
echo $className;
?>
{
struct Tuple {
<?php
foreach ($inputs as $name => $type) {
?>
<?php
echo $type;
?>
<?php
echo $name;
?>
;
<?php
}
// foreach input
?>
// Default Constructor, Copy Constructor, and Copy Assignment are all
// default
Tuple(void) = default;
Tuple(const Tuple &) = default;
Tuple & operator = (const Tuple &) = default;
Tuple(<?php
echo array_template('const {val} & _{key}', ', ', $inputs);
?>
) :
<?php
echo array_template('{key}(_{key})', ', ', $inputs);
?>
{ }
// operator > means that this tuple is "better" than the other tuple.
bool operator > ( const Tuple & other ) const {
<?php
foreach ($mainAtts as $name => $type) {
$op1 = $minimum[$name] ? '<' : '>';
$op2 = !$minimum[$name] ? '<' : '>';
?>
if( <?php
echo $name;
?>
<?php
echo $op1;
?>
other.<?php
echo $name;
?>
)
return true;
else if( <?php
echo $name;
?>
<?php
echo $op2;
?>
other.<?php
echo $name;
?>
)
return false;
<?php
}
// foreach main attribute
?>
return false;
}
bool operator < ( const Tuple& other ) const {
return other > *this;
}
bool operator <= (const Tuple & other ) const {
return ! (*this > other );
}
bool operator >= (const Tuple & other ) const {
return !( other > *this );
}
bool operator == (const Tuple & other ) const {
bool ret = true;
<?php
foreach ($mainAtts as $name => $type) {
?>
ret &= <?php
echo $name;
?>
== other.<?php
echo $name;
?>
;
<?php
}
// foreach main attribute
?>
return ret;
}
}; // struct Tuple
typedef std::vector<Tuple> TupleVector;
public:
class Iterator {
public:
typedef TupleVector::const_iterator iter_type;
private:
iter_type begin;
iter_type end;
public:
Iterator(void) = default;
Iterator(const Iterator &) = default;
Iterator( const iter_type & _begin, const iter_type & _end ) : begin(_begin), end(_end)
{ }
Iterator( const iter_type && _begin, const iter_type && _end ) : begin(_begin), end(_end)
{ }
bool GetNextResult(<?php
echo typed_ref_args($outputs);
?>
) {
if( begin != end ) {
<?php
foreach ($outputs as $name => $type) {
?>
<?php
echo $name;
?>
= begin-><?php
echo $outputMap[$name];
?>
;
<?php
}
?>
begin++;
return true;
}
else {
return false;
}
}
};
private:
uintmax_t __count; // number of tuples covered
TupleVector tuples;
// Iterator for multi output type
Iterator multiIterator;
public:
// Constructor and destructor
<?php
echo $className;
?>
(void) : __count(0), tuples(), multiIterator()
{ }
~<?php
echo $className;
?>
() { }
void AddItem( <?php
echo const_typed_ref_args($inputs);
?>
) {
++__count;
Tuple t(<?php
echo args($inputs);
?>
);
if( tuples.empty() ) {
tuples.push_back(t);
}
else if( t > tuples.front() ) {
tuples.clear();
tuples.push_back(t);
}
else if( t == tuples.front() ) {
tuples.push_back(t);
}
}
void AddState( <?php
echo $className;
?>
& other ) {
if( tuples.size() == 0 ) {
tuples.swap(other.tuples);
}
else if( other.tuples.size() == 0 ) {
// Do nothing
}
else if( tuples.front() > other.tuples.front() ) {
// fast path
}
else if( other.tuples.front() > tuples.front() ) {
tuples.swap(other.tuples);
}
else {
for( Tuple & t : other.tuples ) {
tuples.push_back(t);
}
}
}
void Finalize( void ) {
multiIterator = Iterator(tuples.cbegin(), tuples.cend());
}
bool GetNextResult(<?php
echo typed_ref_args($outputs);
?>
) {
return multiIterator.GetNextResult(<?php
echo args($outputs);
?>
);
}
}; // class <?php
echo $className;
?>
<?php
$system_headers = ['vector', 'algorithm', 'cinttypes'];
if ($debug > 0) {
$system_headers = array_merge($system_headers, ['iostream', 'sstream', 'string']);
}
return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => 'multi', 'system_headers' => $system_headers);
}
function STATE(array $t_args)
{
$type = get_first_key($t_args, ['type', '0']);
grokit_assert(is_gla($type), 'Template argument to STATE must be a valid GLA.');
$type = $type->lookup();
$gContent = '';
$functions = [];
$methods = [];
$className = generate_name('STATE_');
?>
/** Type definition for generic GLA states. This type is only used to
trannsport states withing the same memory space between
operators. The object the state points to MUST be treated like a
const.
Note: this type cannot be read from the disk or written to the
output. A different mechanism will be used for that.
The type in the object must be a hash of the name of the class
used to encode the object. Any function that assumes a certain
type must explicitly verify the correctness of the type.
The object can be manipulated like a basic datatype. STATE objects
do not know how to deallocate the memory they use. Other
mechanisms have to be used to ensure correct deallocation
(acknowledgements of data packets that contain this as members).
**/
class <?php
echo $className;
?>
{
public:
typedef <?php
echo $type;
?>
* pointer_type;
typedef uint64_t hash_type;
private:
pointer_type object;
hash_type type;
public:
<?php
echo $className;
?>
():
object(nullptr),
type(0)
{}
<?php
echo $className;
?>
(pointer_type _object):
object(_object),
type(<?php
echo $type->cHash();
?>
)
{}
pointer_type GetObject() const {
FATALIF(type != <?php
echo $type->cHash();
?>
, "STATE contains incorrect type!");
return object;
}
<?php
$methods[] = ['IsNull', [], 'BASE::BOOL', true];
?>
bool IsNull() const {
return object == nullptr;
}
/** no destructor. object should not be deallocated here */
};
<?php
ob_start();
?>
<?php
$functions[] = ['IsNull', ['@type'], 'BASE::BOOL', true, true];
?>
inline
bool IsNull( const @type & d ) {
return d.IsNull();
}
<?php
$gContent .= ob_get_clean();
?>
<?php
return array('kind' => 'TYPE', 'name' => $className, "complex" => false, 'extras' => ['type' => $type], 'properties' => ['__state__'], 'global_content' => $gContent, 'methods' => $methods, 'functions' => $functions);
}
function CATEGORY(array $t_args)
{
if (array_key_exists('dict', $t_args)) {
$values = $t_args['dict'];
$maxID = 0;
foreach ($values as $id => $val) {
$maxID = \max($id, $maxID);
}
} else {
$old_vals = get_first_key($t_args, ['values', 0]);
$startAt = get_first_key_default($t_args, ['start.at'], 0);
$values = [];
$maxID = $startAt;
foreach ($old_vals as $ind => $val) {
$values[$maxID++] = $val;
}
}
$cardinality = \count($values);
// Add 1 to the cardinality for the invalid id
$storageTypeBits = ceil(log($maxID + 1, 2));
if ($storageTypeBits > 64) {
// This should never happen. PHP would explode processing 2^64 values.
grokit_error("Unable to store {$cardinality} values within 64 bits.");
} else {
if ($storageTypeBits > 32) {
$storageType = 'uint64_t';
$storageBytes = 8;
} else {
if ($storageTypeBits > 16) {
$storageType = 'uint32_t';
$storageBytes = 4;
} else {
if ($storageTypeBits > 8) {
$storageType = 'uint16_t';
$storageBytes = 2;
} else {
$storageType = 'uint8_t';
$storageBytes = 1;
}
}
}
}
$className = generate_name('CATEGORY');
$stringType = lookupType('base::STRING');
$methods = [];
$constructors = [];
$functions = [];
?>
class <?php
echo $className;
?>
{
public:
typedef <?php
echo $storageType;
?>
StorageType;
typedef std::unordered_map<StorageType, std::string> IDToNameMap;
typedef std::unordered_map<std::string, StorageType> NameToIDMap;
static const StorageType InvalidID __attribute__((weak));
private:
static const IDToNameMap idToName __attribute__((weak));
static const NameToIDMap nameToID __attribute__((weak));
// The ID of this categorical variable
StorageType myID;
public:
/* ----- Constructors / Destructor ----- */
<?php
echo $className;
?>
( void );
<?php
$constructors[] = [['base::STRING_LITERAL'], true];
?>
<?php
echo $className;
?>
( const char * );
<?php
$constructors[] = [['base::STRING'], true];
?>
<?php
echo $className;
?>
( const <?php
echo $stringType;
?>
& );
<?php
echo $className;
?>
( const <?php
echo $storageType;
?>
);
<?php
echo $className;
?>
( const <?php
echo $className;
?>
& );
<?php
$constructors[] = [['BASE::NULL'], true];
?>
<?php
echo $className;
?>
( const GrokitNull & );
<?php
echo $className;
?>
& operator =( const <?php
echo $className;
?>
& ) = default;
~<?php
echo $className;
?>
(void) {}
/* ----- Methods ----- */
void FromString( const char * );
<?php
$methods[] = ['ToString', [], 'base::STRING_LITERAL', true];
?>
const char * ToString( void ) const;
StorageType GetID( void ) const;
void SetID( StorageType id );
// Determines whether or not the category is valid.
<?php
$methods[] = ['Invalid', [], 'base::bool', true];
?>
bool Invalid(void) const;
<?php
$methods[] = ['Valid', [], 'base::bool', true];
?>
bool Valid(void) const;
/* ----- Operators ----- */
bool operator ==( const <?php
echo $className;
?>
& ) const;
bool operator !=( const <?php
echo $className;
?>
& ) const;
bool operator <( const <?php
echo $className;
?>
& ) const;
bool operator <=( const <?php
echo $className;
?>
& ) const;
bool operator >( const <?php
echo $className;
?>
& ) const;
bool operator >=( const <?php
echo $className;
?>
& ) const;
// Implicit conversion to storage type
operator <?php
echo $storageType;
?>
() const;
// To/From Json
void toJson( Json::Value & dest ) const;
void fromJson( const Json::Value & src );
};
/* ----- Constructors ----- */
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( void ) :
myID(InvalidID)
{ }
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const char * str ) {
FromString(str);
}
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const <?php
echo $stringType;
?>
& str ) {
FromString(str.ToString());
}
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const <?php
echo $storageType;
?>
val ) :
myID(val)
{ }
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const <?php
echo $className;
?>
& other ) : myID(other.myID)
{ }
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const GrokitNull & nullval ) : myID(InvalidID)
{ }
/* ----- Methods ----- */
inline
void <?php
echo $className;
?>
:: FromString( const char * str ) {
auto it = nameToID.find(str);
if( it != nameToID.end() ) {
myID = it->second;
}
else {
myID = InvalidID;
}
}
inline
const char * <?php
echo $className;
?>
:: ToString( void ) const {
auto it = idToName.find(myID);
if( it != idToName.end() ) {
return it->second.c_str();
}
else {
return "NULL";
}
}
inline
auto <?php
echo $className;
?>
:: GetID( void ) const -> StorageType {
return myID;
}
inline
void <?php
echo $className;
?>
:: SetID( StorageType id ) {
myID = id;
}
inline
bool <?php
echo $className;
?>
:: Valid(void) const {
return idToName.count(myID) > 0;
}
inline
bool <?php
echo $className;
?>
:: Invalid(void) const {
return ! Valid();
}
/* ----- Operators ----- */
inline
bool <?php
echo $className;
?>
:: operator ==( const <?php
echo $className;
?>
& other ) const {
return myID == other.myID;
}
inline
bool <?php
echo $className;
?>
:: operator !=( const <?php
echo $className;
?>
& other ) const {
return myID != other.myID;
}
inline
bool <?php
echo $className;
?>
:: operator <( const <?php
echo $className;
?>
& other ) const {
return myID < other.myID;
}
inline
bool <?php
echo $className;
?>
:: operator >( const <?php
echo $className;
?>
& other ) const {
return myID > other.myID;
}
inline
bool <?php
echo $className;
?>
:: operator <=( const <?php
echo $className;
?>
& other ) const {
return myID <= other.myID;
}
inline
bool <?php
echo $className;
?>
:: operator >=( const <?php
echo $className;
?>
& other ) const {
return myID >= other.myID;
}
// To/From Json
inline
void <?php
echo $className;
?>
:: toJson( Json::Value & dest ) const {
dest = (Json::Int64) myID;
}
inline
void <?php
echo $className;
?>
:: fromJson( const Json::Value & src ) {
myID = (StorageType) src.asInt64();
}
inline
<?php
echo $className;
?>
:: operator <?php
echo $storageType;
?>
() const {
return myID;
}
<?php
ob_start();
$functions[] = ['Hash', ['@type'], 'base::BIGINT', true, true];
?>
template<>
inline
uint64_t Hash(const @type & thing) {
return thing.GetID();
}
inline
void FromString( @type & c, const char * str ) {
c.FromString(str);
}
inline
int ToString( const @type & c, char * buffer ) {
const char * str = c.ToString();
strcpy( buffer, str);
int len = strlen(buffer);
return len + 1;
}
inline
void ToJson( const @type & src, Json::Value & dest ) {
src.toJson(dest);
}
inline
void FromJson( const Json::Value & src, @type & dest ) {
dest.fromJson(src);
}
<?php
$functions[] = ['IsNull', ['@type'], 'BASE::BOOL', true, true];
?>
inline
bool IsNull( const @type c ) {
return c.Invalid();
}
<?php
$globalContents = ob_get_clean();
?>
// Initialize static values
const <?php
echo $className;
?>
::IDToNameMap <?php
echo $className;
?>
:: idToName = { <?php
echo array_template('{{key},"{val}"}', ',', $values);
?>
};
const <?php
echo $className;
?>
::NameToIDMap <?php
echo $className;
?>
:: nameToID = { <?php
echo array_template('{"{val}",{key}}', ',', $values);
?>
};
const <?php
echo $className;
?>
::StorageType <?php
echo $className;
?>
:: InvalidID = std::numeric_limits<<?php
echo $className;
?>
::StorageType>::max();
<?php
return ['kind' => 'TYPE', 'name' => $className, 'properties' => ['categorical'], 'extras' => ['cardinality' => $cardinality, 'size.bytes' => $storageBytes], 'binary_operators' => ['==', '!=', '<', '>', '<=', '>='], 'system_headers' => ['cinttypes', 'unordered_map', 'string', 'cstring', 'limits'], 'global_content' => $globalContents, 'complex' => false, 'methods' => $methods, 'constructors' => $constructors, 'functions' => $functions, 'describe_json' => DescribeJson('factor', DescribeJsonStatic(['levels' => $values]))];
}
function Multiplexer(array $t_args, array $inputs, array $outputs)
{
$className = generate_name('Multiplexer');
if (\count($inputs) == 0) {
grokit_assert(array_key_exists('input', $t_args), 'No inputs specified for Multiplexer');
$inputs = $t_args['input'];
foreach ($t_args['inputs'] as $name => &$type) {
if (is_identifier($type)) {
$type = lookupType(strval($type));
}
grokit_assert(is_datatype($type), 'Only types may be specified as inputs to Multiplexer.');
}
$inputs = ensure_valid_names($inputs, 'multi_input');
}
$glas = get_first_key($t_args, ['glas', 0]);
grokit_assert(\count($glas) > 0, 'No GLAs specified for Multiplexer.');
$myGLAs = [];
$glaInputs = [];
$glaOutputs = [];
$resultType = 'multi';
$usedOutputs = [];
$libraries = [];
$glaGenStates = [];
$glaReqStates = [];
$configurable = false;
$constArgs = [];
$genStates = [];
$reqStates = [];
$iterable = null;
foreach ($glas as $name => $glaInfo) {
grokit_assert(is_array($glaInfo), 'Template argument \'glas\' must be an array');
grokit_assert(array_key_exists('gla', $glaInfo), 'No GLA given for glas[' . $name . ']');
grokit_assert(array_key_exists('inputs', $glaInfo), 'No inputs given for glas[' . $name . ']');
grokit_assert(array_key_exists('outputs', $glaInfo), 'No outputs given for glas[' . $name . ']');
$gla = $glaInfo['gla'];
$glaInAtts = $glaInfo['inputs'];
$glaOutAtts = $glaInfo['outputs'];
grokit_assert(is_gla($gla), 'Non-GLA given for glas[' . $name . '][gla]');
grokit_assert(is_array($glaInAtts), 'Non-array given for inputs for gla ' . $name);
grokit_assert(is_array($glaOutAtts), 'Non-array given for outputs for gla ' . $name);
$glaInAtts = array_map('strval', $glaInAtts);
$glaOutAtts = array_map('strval', $glaOutAtts);
$glaName = "innerGLA_" . $name;
$glaInputs[$glaName] = [];
$glaOutputs[$glaName] = [];
foreach ($glaInAtts as $att) {
grokit_assert(array_key_exists($att, $inputs), 'Input ' . $att . ' for GLA ' . $name . ' not found in inputs');
$glaInputs[$glaName][$att] = $inputs[$att];
}
foreach ($glaOutAtts as $att) {
grokit_assert(array_key_exists($att, $outputs), 'Output ' . $att . ' for GLA ' . $name . ' not found in outputs');
grokit_assert(!in_array($att, $usedOutputs), 'Output ' . $att . ' used by multiple GLAs');
$usedOutputs[] = $att;
$glaOutputs[$glaName][$att] = $outputs[$att];
}
//fwrite(STDERR, "Inputs for GLA " . $glaName . ": " . print_r($glaInputs[$glaName], true) . PHP_EOL );
//fwrite(STDERR, "Outputs for GLA " . $glaName . ": " . print_r($glaOutputs[$glaName], true) . PHP_EOL );
$gla = $gla->apply($glaInputs[$glaName], $glaOutputs[$glaName]);
$myGLAs[$glaName] = $gla;
$glaRez[$glaName] = get_first_value($gla->result_type(), ['multi', 'single', 'state']);
$libraries = array_merge($libraries, $gla->libraries());
if ($glaRez[$glaName] == 'state') {
grokit_assert(\count($glaOutputs[$glaName]) == 1, "GLA {$glaName} is produced as state, and thus must have exactly 1 output.");
$stateType = lookupType('base::STATE', ['type' => $gla]);
$glaOutputs[$glaName] = array_combine(array_keys($glaOutputs[$glaName]), [$stateType]);
} else {
grokit_assert(\count($glaOutputs[$glaName]) == \count($gla->output()), 'GLA ' . $glaName . ' produces different number of outputs than expected');
$glaOutputs[$glaName] = array_combine(array_keys($glaOutputs[$glaName]), $gla->output());
}
// Set types for our output
foreach ($glaOutputs[$glaName] as $attName => $type) {
$outputs[$attName] = $type;
}
if (is_null($iterable)) {
$iterable = $gla->iterable();
} else {
grokit_assert($iterable == $gla->iterable(), 'Multiplexer does not support mixing iterable and non-iterable GLAs');
}
$glaReqStates[$glaName] = $gla->req_states();
foreach ($gla->req_states() as $rstate) {
$reqStates[] = $rstate;
}
$glaGenStates[$glaName] = $gla->state();
// TODO: Support constant states
grokit_assert(!$gla->has_state(), 'Multiplexer currently does not support constant states.');
}
$libraries = array_unique($libraries);
$extra = ['glas' => $myGLAs];
?>
class <?php
echo $className;
?>
{
<?php
foreach ($myGLAs as $name => $type) {
?>
<?php
echo $type;
?>
<?php
echo $name;
?>
;
<?php
}
// foreach inner gla
?>
class Iterator {
bool _gotResultsOnce;
bool _valid;
<?php
foreach ($myGLAs as $name => $type) {
?>
<?php
echo $type;
?>
* it_<?php
echo $name;
?>
;
<?php
}
// foreach inner gla
?>
public:
Iterator(void) : _gotResultsOnce(false), _valid(false),
<?php
echo array_template('it_{key}(nullptr)', ', ', $myGLAs);
?>
{ }
Iterator(<?php
echo typed_ref_args($myGLAs);
?>
) : _gotResultsOnce(false), _valid(true),
<?php
echo array_template('it_{key}(&{key})', ', ', $myGLAs);
?>
{
<?php
foreach ($myGLAs as $name => $type) {
if ($glaRez[$name] == 'multi') {
?>
<?php
echo $name;
?>
.Finalize();
<?php
}
// if inner GLA is multi
}
// foreach inner gla
?>
}
Iterator( const Iterator & other) = default;
~Iterator() { }
bool GetNextResult( <?php
echo typed_ref_args($outputs);
?>
) {
FATALIF(!_valid, "Tried to get results from an invalid iterator.");
bool ret = !_gotResultsOnce;
_gotResultsOnce = true;
<?php
foreach ($myGLAs as $name => $type) {
if ($glaRez[$name] == 'multi') {
?>
ret |= it_<?php
echo $name;
?>
->GetNextResult(<?php
echo args($glaOutputs[$name]);
?>
);
<?php
}
// if inner GLA is multi
}
// foreach inner gla
?>
if( ret ) {
<?php
foreach ($myGLAs as $name => $type) {
if ($glaRez[$name] == 'single') {
?>
it_<?php
echo $name;
?>
->GetResult(<?php
echo args($glaOutputs[$name]);
?>
);
<?php
} else {
if ($glaRez[$name] == 'state') {
$stateVar = array_keys($glaOutputs[$name])[0];
$stateType = $glaOutputs[$name][$stateVar];
?>
<?php
echo $stateVar;
?>
= <?php
echo $stateType;
?>
(it_<?php
echo $name;
?>
);
<?php
}
}
// if inner GLA is state
}
// foreach inner gla
?>
}
return ret;
}
};
Iterator multiIterator;
public:
<?php
echo $className;
?>
() { }
~<?php
echo $className;
?>
() { }
void AddItem(<?php
echo const_typed_ref_args($inputs);
?>
) {
// Call AddItem individually on each GLA.
<?php
foreach ($myGLAs as $gName => $gType) {
?>
<?php
echo $gName;
?>
.AddItem(<?php
echo args($glaInputs[$gName]);
?>
);
<?php
}
// foreach inner gla
?>
}
void AddState( <?php
echo $className;
?>
& other ) {
// Call AddState individually on each GLA.
<?php
foreach ($myGLAs as $gName => $gType) {
?>
<?php
echo $gName;
?>
.AddState(other.<?php
echo $gName;
?>
);
<?php
}
// foreach inner gla
?>
}
void Finalize() {
multiIterator = Iterator(<?php
echo args($myGLAs);
?>
);
}
bool GetNextResult(<?php
echo typed_ref_args($outputs);
?>
) {
return multiIterator.GetNextResult(<?php
echo args($outputs);
?>
);
}
void GetResult(<?php
echo typed_ref_args($outputs);
?>
) {
Finalize();
GetNextResult(<?php
echo args($outputs);
?>
);
}
<?php
foreach (array_keys($myGLAs) as $index => $name) {
?>
const <?php
echo $myGLAs[$name];
?>
& GetGLA<?php
echo $index;
?>
() const {
return <?php
echo $name;
?>
;
}
<?php
}
?>
};
<?php
return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => $resultType, 'libraries' => $libraries, 'configurable' => $configurable, 'extra' => $extra);
}
function HEX($t_args)
{
// The number of bytes encoded per string.
$size = get_first_key($t_args, ['size', 0]);
$className = 'Hex' . $size;
$systemHeaders = ['cstdio', 'cstring'];
$userHeaders = [];
$libHeaders = [];
$libraries = [];
$constructors = [];
$methods = [];
$functions = [];
$binaryOperators = [];
$unaryOperators = [];
$globalContent = '';
$complex = false;
$properties = [];
$extra = ['size.bytes' => $size];
$describeJson = DescribeJson('hex', DescribeJsonStatic(['size' => $size]));
$globalContent = '';
?>
class <?php
echo $className;
?>
{
public:
// Lookup table per pair of hex digits.
static constexpr const char lookup[256] __attribute__((weak)) = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
static constexpr const char to_char[16] __attribute__((weak)) = {
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'
};
// The number of bytes encoded.
static const constexpr size_t kSize = <?php
echo $size;
?>
;
// The length of decoded strings.
static const constexpr size_t kLength = 2 * kSize;
private:
// The binary data corresponding to the encoded string.
std::array<char, kSize> bytes;
public:
<?php
$constructors[] = [[], true];
?>
<?php
echo $className;
?>
();
<?php
$constructors[] = [['BASE::NULL'], true];
?>
<?php
echo $className;
?>
(const GrokitNull& nullval);
<?php
$constructors[] = [['BASE::STRING_LITERAL'], true];
?>
<?php
echo $className;
?>
(const char* str);
<?php
$methods[] = ['IsValid', [], 'BASE::BOOL', true];
?>
bool IsValid(void) const;
<?php
$methods[] = ['IsNull', [], 'BASE::BOOL', true];
?>
bool IsNull(void) const;
<?php
$binaryOperators[] = '==';
?>
bool operator ==(const <?php
echo $className;
?>
& other) const;
<?php
$binaryOperators[] = '!=';
?>
bool operator !=(const <?php
echo $className;
?>
& other) const;
void FromString(const char* str);
int ToString(char* buffer) const;
void FromJson(const Json::Value& src);
void ToJson(Json::Value& dest) const;
};
// Allocate storage for static variables
constexpr const char <?php
echo $className;
?>
::lookup[256];
constexpr const char <?php
echo $className;
?>
::to_char[16];
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
() {
bytes.fill(0);
}
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
(const GrokitNull& nullval) {
bytes.fill(0);
}
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
(const char* str) {
this->FromString(str);
}
inline bool <?php
echo $className;
?>
::operator ==(const <?php
echo $className;
?>
& other) const {
return std::memcmp(bytes.data(), other.bytes.data(), kSize);
}
inline bool <?php
echo $className;
?>
::operator !=(const <?php
echo $className;
?>
& other) const {
return !(*this == other);
}
inline void <?php
echo $className;
?>
::FromString(const char* str) {
size_t len = strlen(str);
FATALIF(len != kLength, "Incorrectly sized string, expected %zu, got %zu: %s",
kLength, len, str);
for (size_t i = 0; i < kSize; i++) {
char a = lookup[str[2 * i]];
char b = lookup[str[2 * i + 1]];
FATALIF(a < 0 || b < 0, "Illegal hex character: %c%c",
str[2 * i], str[2 * i + 1]);
bytes[i] = (a << 4) | b;
}
}
inline int <?php
echo $className;
?>
::ToString(char* buffer) const {
for (size_t i = 0; i < kSize; i++) {
char byte = bytes[i];
buffer[2 * i] = to_char[(byte >> 4) & 0x0F];
buffer[2 * i + 1] = to_char[byte & 0x0F];
}
buffer[kLength] = '\0';
return 1 + kLength;
}
inline void <?php
echo $className;
?>
::FromJson(const Json::Value& src) {
this->FromString(src.asCString());
}
inline void <?php
echo $className;
?>
::ToJson(Json::Value& dest) const {
char buffer[kLength + 1];
this->ToString(buffer);
dest = buffer;
}
<?php
ob_start();
?>
inline void FromString(@type& data, const char* buffer) {
data.FromString(buffer);;
}
inline int ToString(const @type& data, char* buffer) {
return data.ToString(buffer);
}
inline void FromJson(const Json::Value& src, @type& dest) {
dest.FromJson(src);
}
inline void ToJson(const @type& src, Json::Value& dest) {
src.ToJson(dest);
}
<?php
$globalContent .= ob_get_clean();
?>
<?php
return ['kind' => 'TYPE', 'name' => $className, 'complex' => $complex, 'system_headers' => $systemHeaders, 'user_headers' => $userHeaders, 'lib_headers' => $libHeaders, 'libraries' => $libraries, 'binary_operators' => $binaryOperators, 'unary_operators' => $unaryOperators, 'global_content' => $globalContent, 'constructors' => $constructors, 'methods' => $methods, 'functions' => $functions, 'libraries' => $libraries, 'properties' => $properties, 'extras' => $extra, 'describe_json' => $describeJson];
}
function Segmenter(array $t_args, array $input, array $output, array $given_states)
{
$resType = ['fragment', 'multi'];
$system_headers = ['array', 'vector', 'memory', 'cinttypes', 'unordered_map'];
$user_headers = ['HashFunctions.h'];
$lib_headers = [];
$preferFragment = get_default($t_args, 'inner.prefer.fragment', false);
$wantedRes = $preferFragment ? ['fragment', 'multi'] : ['multi', 'fragment'];
$nInputs = \count($input);
grokit_assert($nInputs > 1, 'Segmenter: Not enough inputs specified!');
$keyName = array_keys($input)[0];
$keyType = array_get_index($input, 0);
$innerInputs = array_slice($input, 1, $nInputs - 1, true);
$gla = get_first_key($t_args, ['gla', 'GLA', 0]);
grokit_assert(is_gla($gla), 'Segmenter: [gla] argument must be a valid GLA');
$gla = $gla->apply($innerInputs, $output, $given_states);
$n_passes = get_default($t_args, 'passes', 1);
grokit_assert(is_int($n_passes), 'Segmenter: [passes] argument must be an integer');
grokit_assert($n_passes > 0, 'Segmenter: [passes] argument must be > 0');
$libraries = $gla->libraries();
$innerRes = get_first_value($gla->result_type(), $wantedRes);
$innerInputs = $gla->input();
$innerOutput = $gla->output();
$input = array_merge([$keyName => $keyType], $innerInputs);
$output = $innerOutput;
$segments = get_default($t_args, 'segments', 64);
$constState = lookupResource('BASE::SegmenterState', ['gla' => $gla, 'passes' => $n_passes, 'segments' => $segments]);
$className = generate_name('Segmenter_');
$savedArgs = [];
$cArgs = [];
$innerCArgs = [];
if ($gla->configurable()) {
$savedArgs['json_init'] = 'Json::Value';
$cArgs['json_init'] = 'Json::Value';
$innerCArgs[] = 'json_init';
}
$cArgs['const_state'] = $constState;
if ($gla->has_state()) {
$innerCArgs[] = 'constState.inner_cstate';
}
$cstStr = \count($innerCArgs) > 0 ? '(' . implode(',', $innerCArgs) . ')' : '';
grokit_assert(!$gla->iterable(), 'Segementer does not support iterable GLAs');
$iterable = $n_passes > 1;
?>
class <?php
echo $className;
?>
{
private:
using ConstantState = <?php
echo $constState;
?>
;
using SplitState = ConstantState::SplitState;
static constexpr const size_t NUM_STATES = SplitState::NUM_STATES;
using InnerGLA = <?php
echo $gla;
?>
;
using InnerGLAPtr = std::unique_ptr<InnerGLA>;
using GLA_Array = std::array<InnerGLAPtr, NUM_STATES>;
public:
using size_type = std::size_t;
<?php
if ($innerRes == 'fragment') {
?>
class Iterator {
private:
InnerGLA * gla;
int fragmentNum;
InnerGLA::Iterator * innerIter;
public:
Iterator( InnerGLA * _gla, int _fragmentNum, int _innerFrag ) :
gla(_gla), fragmentNum(_fragmentNum),
innerIter(nullptr)
{
innerIter = gla->Finalize(_innerFrag);
}
~Iterator(void) {
if( innerIter != nullptr ) {
delete innerIter;
innerIter = nullptr;
}
}
bool GetNextResult( <?php
echo typed_ref_args($gla->output());
?>
) {
return innerIter->GetNextResult(<?php
echo args($gla->output());
?>
);
}
int FragmentNumber() {
return fragmentNum;
}
};
<?php
} else {
// if inner result type is fragment
?>
class Iterator {
private:
InnerGLA * gla;
int fragmentNum;
public:
Iterator( InnerGLA * _gla, int fragNo ) : gla(_gla), fragmentNum(fragNo) {
gla->Finalize();
}
~Iterator(void) { }
bool GetNextResult( <?php
echo typed_ref_args($gla->output());
?>
) {
return gla->GetNextResult(<?php
echo args($gla->output());
?>
);
}
int FragmentNumber() {
return fragmentNum;
}
};
<?php
}
// if inner result type is multi
?>
private:
const ConstantState & constState;
GLA_Array localState;
// Iteration state for multi result type
int numFrags;
int multiFragNo;
Iterator * multiIter;
<?php
if ($innerRes == 'fragment') {
?>
using frag_info = std::pair<int, int>;
using frag_map_t = std::unordered_map<int, frag_info>;
frag_map_t fragMap;
<?php
}
?>
<?php
foreach ($savedArgs as $name => $type) {
?>
const <?php
echo $type;
?>
<?php
echo $name;
?>
;
<?php
}
// foreach saved arg
?>
public:
// Constructor
<?php
echo $className;
?>
( <?php
echo const_typed_ref_args($cArgs);
?>
) :
constState(const_state)
, localState()
, numFrags(0)
, multiFragNo(0)
, multiIter(nullptr)
<?php
if ($innerRes == 'fragment') {
?>
, fragMap()
<?php
}
foreach ($savedArgs as $name => $type) {
?>
, <?php
echo $name;
?>
(<?php
echo $name;
?>
)
<?php
}
// foreach constructor arg to save
?>
{
for( auto & elem : localState ) {
elem.reset(new InnerGLA<?php
echo $cstStr;
?>
);
}
}
void AddItem( <?php
echo const_typed_ref_args($input);
?>
) {
uint64_t hashVal = CongruentHash(Hash(<?php
echo $keyName;
?>
), H_b + 1);
uint64_t passNum = (hashVal / NUM_STATES) % ConstantState::N_PASSES;
uint64_t segNum = hashVal % NUM_STATES;
<?php
if ($n_passes > 1) {
?>
if( passNum != constState.pass ) {
return;
}
<?php
}
// more than 1 pass
?>
localState[segNum]->AddItem(<?php
echo args($innerInputs);
?>
);
}
void ChunkBoundary(void) {
// Merge local states into the global state
SplitState & globalStates = constState.segments;
int theseAreOk[NUM_STATES];
for( int i = 0; NUM_STATES > i; i++ ) {
theseAreOk[i] = 1;
}
int segsLeft = NUM_STATES;
while( segsLeft > 0 ) {
InnerGLA * checkedOut = nullptr;
int whichOne = globalStates.CheckOutOne( theseAreOk, checkedOut );
if( checkedOut == NULL ) {
checkedOut = new InnerGLA<?php
echo $cstStr;
?>
;
}
checkedOut->AddState( *(localState[whichOne]) );
globalStates.CheckIn( whichOne, checkedOut );
theseAreOk[whichOne] = 0;
segsLeft--;
}
// Re-initialize the local states
for( auto & elem : localState ) {
<?php
if ($gla->is('resettable')) {
?>
elem->Reset();
<?php
} else {
// if resettable
?>
elem.reset(new InnerGLA<?php
echo $cstStr;
?>
);
<?php
}
// if not resettable
?>
}
}
void AddState( <?php
echo $className;
?>
& o ) {
// Do nothing
}
void Finalize() {
SplitState & globalStates = constState.segments;
if( multiIter != nullptr)
delete multiIter;
multiFragNo = 0;
<?php
if ($innerRes == 'fragment') {
?>
frag_info fInfo = fragMap[multiFragNo];
multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo,
fInfo.second);
<?php
} else {
?>
multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo);
<?php
}
?>
}
bool GetNextResult(<?php
echo typed_ref_args($output);
?>
) {
bool gotResult = false;
SplitState & globalStates = constState.segments;
while( (multiFragNo < numFrags && multiIter != nullptr) && !gotResult ) {
gotResult = multiIter->GetNextResult(<?php
echo args($output);
?>
);
if( !gotResult ) {
multiFragNo++;
delete multiIter;
if( numFrags > multiFragNo ) {
<?php
if ($innerRes == 'fragment') {
?>
frag_info fInfo = fragMap[multiFragNo];
multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo,
fInfo.second);
<?php
} else {
?>
multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo);
<?php
}
?>
} else {
multiIter = nullptr;
}
}
}
return gotResult;
}
int GetNumFragments(void) {
<?php
if ($innerRes == 'fragment') {
?>
SplitState & globalStates = constState.segments;
numFrags = 0;
for (int i = 0; i < NUM_STATES; i++) {
int curFrags = globalStates.Peek(i)->GetNumFragments();
for (int curFrag = 0; curFrag < curFrags; curFrag++) {
fragMap[numFrags] = frag_info(i, curFrag);
numFrags++;
}
}
<?php
} else {
?>
numFrags = NUM_STATES;
<?php
}
?>
return numFrags;
}
Iterator * Finalize( int fragment ) {
SplitState & globalStates = constState.segments;
<?php
if ($innerRes == 'fragment') {
?>
frag_info info = fragMap[fragment];
return new Iterator(globalStates.Peek(info.first), fragment, info.second);
<?php
} else {
?>
return new Iterator(globalStates.Peek(fragment), fragment);
<?php
}
?>
}
bool GetNextResult( Iterator * it, <?php
echo typed_ref_args($output);
?>
) {
bool ret = it->GetNextResult(<?php
echo args($output);
?>
);
return ret;
}
<?php
if ($iterable) {
?>
bool ShouldIterate( ConstantState & modible ) {
modible.pass++;
return modible.pass < ConstantState::N_PASSES;
}
void PostFinalize() {
constState.segments.Reset();
}
<?php
}
// iterable
?>
<?php
if ($gla->is('finite container')) {
?>
size_type size() {
SplitState & globalStates = constState.segments;
size_type s = 0;
for( int i = 0; NUM_STATES > i; i++ ) {
InnerGLA * ptr = globalStates.Peek(i);
s += ptr->size();
}
return s;
}
size_type size(int frag) {
SplitState & globalStates = constState.segments;
return globalStates.Peek(frag)->size();
}
<?php
}
// if the gla is a container
?>
};
typedef <?php
echo $className;
?>
::Iterator <?php
echo $className;
?>
_Iterator;
<?php
return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $system_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'input' => $input, 'output' => $output, 'result_type' => $resType, 'generated_state' => $constState, 'required_states' => $gla->req_states(), 'chunk_boundary' => true, 'configurable' => $gla->configurable(), 'iterable' => $iterable, 'post_finalize' => $iterable, 'intermediates' => true];
}
function FACTOR(array $t_args)
{
$rawDict = get_first_key($t_args, ['dictionary', 'dict', 0]);
// Double the quotes so that we escape them in SQLite, and add backslashes
// to them so that we escape them in C++.
$dict = addcslashes(\grokit\doubleChars($rawDict, '"'), '"\\');
$cardinality = \grokit\dictionarySize($rawDict);
$storageBytes = get_first_key_default($t_args, ['bytes', 1], 2);
$cardBytes = $cardinality > 0 ? intval(ceil(log($cardinality, 256))) : 1;
$storageBytes = $cardBytes > $storageBytes ? $cardBytes : $storageBytes;
switch ($storageBytes) {
case 1:
$storageType = 'uint8_t';
break;
case 2:
$storageType = 'uint16_t';
break;
case 4:
$storageType = 'uint32_t';
break;
case 8:
$storageType = 'uint64_t';
break;
default:
grokit_error('Unsupported # of bytes (' . $storageBytes . ') given for FACTOR, only 1, 2, 4, and 8 supported.');
}
$className = generate_name('FACTOR_' . ensure_identifier($dict));
$stringType = lookupType('base::STRING');
$globalContent = '';
$methods = [];
$constructors = [];
$functions = [];
?>
class <?php
echo $className;
?>
{
public:
typedef <?php
echo $storageType;
?>
StorageType;
static const char * DictionaryName __attribute__((weak));
static const StorageType InvalidID __attribute__((weak));
static const StorageType MaxID __attribute__((weak));
static const Dictionary & globalDictionary __attribute__((weak));
public:
/* ----- Members ----- */
// The ID of this Factor;
StorageType myID;
/* ----- Constructors / Destructors ----- */
// Default constructor
<?php
echo $className;
?>
( void );
// Constructor from null (same as default)
<?php
echo $className;
?>
( const GrokitNull & );
// Constructor from C strings / string literals
<?php
$constructors[] = [['base::STRING_LITERAL'], true];
?>
<?php
echo $className;
?>
( const char * );
// Constructor from Grokit STRING type.
<?php
$constructors[] = [['base::STRING'], true];
?>
<?php
echo $className;
?>
( const <?php
echo $stringType;
?>
& );
// Constructor from storage type
<?php
echo $className;
?>
( const StorageType );
// Copy constructor and copy assignment
// These can both be default
<?php
echo $className;
?>
( const <?php
echo $className;
?>
& ) = default;
<?php
echo $className;
?>
& operator =( const <?php
echo $className;
?>
& ) = default;
// Destructor
~<?php
echo $className;
?>
() { }
/* ----- Methods ----- */
// Standard FromString method
void FromString( const char * );
// FromString method used when building the dictionaries.
void FromString( const char *, Dictionary & );
// Looks up the factor in the global dictionary and returns the string
<?php
$methods[] = ['ToString', [], 'base::STRING_LITERAL', true];
?>
const char * ToString( void ) const;
// Returns the ID of the Factor.
StorageType GetID( void ) const;
// Returns whether or not the Factor is valid.
<?php
$methods[] = ['Valid', [], 'base::bool', true];
?>
bool Valid( void ) const;
<?php
$methods[] = ['Invalid', [], 'base::bool', true];
?>
bool Invalid( void ) const;
// Translate the content
void Translate( const Dictionary::TranslationTable& );
void toJson( Json::Value & dest ) const;
void fromJson( const Json::Value & src );
/* ----- Operators ----- */
// The dictionary keeps track of what the sorted order of the strings is.
// These methods are based on the lexicographical ordering of the strings
// the factors represent
bool operator ==( const <?php
echo $className;
?>
& ) const;
bool operator !=( const <?php
echo $className;
?>
& ) const;
bool operator <( const <?php
echo $className;
?>
& ) const;
bool operator <=( const <?php
echo $className;
?>
& ) const;
bool operator >( const <?php
echo $className;
?>
& ) const;
bool operator >=( const <?php
echo $className;
?>
& ) const;
// Implicit conversion to storage type
operator StorageType () const;
};
// Static member initialization
const <?php
echo $className;
?>
::StorageType <?php
echo $className;
?>
::InvalidID = std::numeric_limits<StorageType>::max();
const <?php
echo $className;
?>
::StorageType <?php
echo $className;
?>
::MaxID = <?php
echo $className;
?>
::InvalidID - 1;
const char * <?php
echo $className;
?>
::DictionaryName = "<?php
echo $dict;
?>
";
const Dictionary & <?php
echo $className;
?>
::globalDictionary = Dictionary::GetDictionary(<?php
echo $className;
?>
::DictionaryName);
/* ----- Constructors ----- */
// Default constructor
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( void ):
myID(InvalidID)
{}
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const GrokitNull & nullval ):
myID(InvalidID)
{ }
// Constructor from C strings / string literals
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const char * str ) {
FromString(str);
}
// Constructor from Grokit STRING type
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const <?php
echo $stringType;
?>
& str ) {
FromString(str.ToString());
}
// Constructor from storage type
inline
<?php
echo $className;
?>
:: <?php
echo $className;
?>
( const <?php
echo $storageType;
?>
id ):
myID(id)
{ }
/* ----- Methods ----- */
inline
auto <?php
echo $className;
?>
:: GetID(void) const -> StorageType {
return myID;
}
// Standard FromString method
inline
void <?php
echo $className;
?>
:: FromString( const char * str ) {
// Global dictionary will return InvalidID if not found
myID = globalDictionary.Lookup(str, InvalidID );
}
// FromString method used when building the dictionaries
inline
void <?php
echo $className;
?>
:: FromString( const char * str, Dictionary & localDict ) {
// First check if we are in the local dictionary
myID = localDict.Lookup(str, InvalidID );
if( myID != InvalidID )
return;
// Next check if we are in the global dictionary
myID = globalDictionary.Lookup(str, InvalidID );
if( myID != InvalidID )
return;
// Add a new entry to the local dictionary.
// The dictionary should throw an error if the new ID is greater than
// MaxID.
myID = localDict.Insert( str, MaxID );
}
// Looks up the factor in the global dictionary and returns the string
inline
const char * <?php
echo $className;
?>
:: ToString( void ) const {
return globalDictionary.Dereference(myID);
}
// Determine whether or not the factor is valid
inline
bool <?php
echo $className;
?>
:: Valid( void ) const {
return myID != InvalidID;
}
inline
bool <?php
echo $className;
?>
:: Invalid(void) const {
return myID == InvalidID;
}
// Translate the content
inline
void <?php
echo $className;
?>
:: Translate( const Dictionary::TranslationTable & tbl ) {
auto it = tbl.find(myID);
if( it != tbl.end() ) {
myID = it->second;
}
}
inline
void <?php
echo $className;
?>
:: toJson( Json::Value & dest ) const {
dest = (Json::Int64) myID;
}
inline
void <?php
echo $className;
?>
:: fromJson( const Json::Value & src ) {
myID = (StorageType) src.asInt64();
}
/* ----- Operators ----- */
inline
bool <?php
echo $className;
?>
:: operator ==( const <?php
echo $className;
?>
& o ) const {
return myID == o.myID;
}
inline
bool <?php
echo $className;
?>
:: operator !=( const <?php
echo $className;
?>
& o ) const {
return myID != o.myID;
}
inline
bool <?php
echo $className;
?>
:: operator <( const <?php
echo $className;
?>
& o ) const {
return Valid() && o.Valid() && globalDictionary.Compare(myID, o.myID) < 0;
}
inline
bool <?php
echo $className;
?>
:: operator <=( const <?php
echo $className;
?>
& o ) const {
return Valid() && o.Valid() && globalDictionary.Compare(myID, o.myID) <= 0;
}
inline
bool <?php
echo $className;
?>
:: operator >( const <?php
echo $className;
?>
& o ) const {
return Valid() && o.Valid() && globalDictionary.Compare(myID, o.myID) > 0;
}
inline
bool <?php
echo $className;
?>
:: operator >=( const <?php
echo $className;
?>
& o ) const {
return Valid() && o.Valid() && globalDictionary.Compare(myID, o.myID) >= 0;
}
// Implicit conversion to storage type
inline
<?php
echo $className;
?>
:: operator StorageType() const {
return myID;
}
<?php
ob_start();
// Global functions
?>
inline
void FromString( @type & f, const char * str ) {
f.FromString(str);
}
inline
void FromString( @type & f, const char * str, Dictionary & localDict ) {
f.FromString(str, localDict);
}
inline
int ToString( const @type & f, char * buffer ) {
const char * str = f.ToString();
strcpy(buffer, str);
return strlen(buffer) + 1;
}
<?php
$functions[] = ['Hash', ['@type'], 'base::BIGINT', true, true];
?>
template<>
inline
uint64_t Hash( const @type & x ) {
return x.GetID();
}
inline
void ToJson( const @type & src, Json::Value & dest ) {
src.toJson(dest);
}
inline
void FromJson( const Json::Value & src, @type & dest ) {
dest.fromJson(src);
}
<?php
$functions[] = ['IsNull', ['@type'], 'BASE::BOOL', true, true];
?>
inline
bool IsNull( const @type f ) {
return f.Invalid();
}
<?php
$globalContent .= ob_get_clean();
?>
<?php
// Function to get the dictionary at runtime.
$describeInfoJson = function ($var, $myType) {
?>
<?php
echo $var;
?>
["levels"] = Json::Value(Json::arrayValue);
for( auto it = <?php
echo $myType;
?>
::globalDictionary.cbegin(); it != <?php
echo $myType;
?>
::globalDictionary.cend(); it++ ) {
<?php
echo $var;
?>
["levels"][it->first] = it->second;
}
<?php
};
return ['kind' => 'TYPE', 'name' => $className, 'dictionary' => $dict, 'system_headers' => ['limits', 'cstring', 'cinttypes'], 'user_headers' => ['Dictionary.h', 'DictionaryManager.h', 'ColumnIteratorDict.h'], 'properties' => ['categorical'], 'extras' => ['cardinality' => $cardinality, 'size.bytes' => $storageBytes], 'binary_operators' => ['==', '!=', '<', '>', '<=', '>='], 'global_content' => $globalContent, 'complex' => 'ColumnIteratorDict< @type >', 'methods' => $methods, 'constructors' => $constructors, 'functions' => $functions, 'describe_json' => DescribeJson('factor', $describeInfoJson)];
}
function CATEGORYSET($t_args)
{
// The dictionary of the associated factor.
$dictionary = get_first_key($t_args, ['dictionary', 'dict', 0]);
$values = array_values($dictionary);
// The component types used.
$category = lookupType('category', ['dict' => $dictionary]);
$bitset = lookupType('bitset', ['values' => $values]);
$size = $bitset->get('size.bytes');
// The name of the object type.
$className = 'CategorySet' . $size;
$systemHeaders = ['cstdio', 'cstring'];
$userHeaders = [];
$libHeaders = [];
$libraries = [];
$constructors = [];
$methods = [];
$functions = [];
$binaryOperators = [];
$unaryOperators = [];
$globalContent = '';
$complex = false;
$properties = [];
$extra = ['size.bytes' => $size];
$describeJson = DescribeJson('categoryset', DescribeJsonStatic(['levels' => $values]));
$globalContent = '';
?>
class <?php
echo $className;
?>
{
public:
using Category = <?php
echo $category;
?>
;
using BitSet = <?php
echo $bitset;
?>
;
using StorageType = BitSet::StorageType;
private:
// The binary data corresponding to the encoded string.
BitSet data;
public:
<?php
$constructors[] = [[], true];
?>
<?php
echo $className;
?>
();
<?php
$constructors[] = [['BASE::NULL'], true];
?>
<?php
echo $className;
?>
(const GrokitNull& null);
<?php
$constructors[] = [['BASE::STRING_LITERAL'], true];
?>
<?php
echo $className;
?>
(const char* str);
<?php
echo $className;
?>
(const BitSet& data);
<?php
echo $className;
?>
(const <?php
echo $className;
?>
& other);
<?php
$methods[] = ['IsEmpty', [], 'BASE::BOOL', true];
?>
bool IsEmpty() const;
<?php
echo $className;
?>
& operator =(const <?php
echo $className;
?>
& other) = default;
<?php
$binaryOperators[] = '==';
?>
bool operator ==(const <?php
echo $className;
?>
& other) const;
<?php
$binaryOperators[] = '!=';
?>
bool operator !=(const <?php
echo $className;
?>
& other) const;
<?php
$binaryOperators[] = '<';
?>
bool operator <(const <?php
echo $className;
?>
& other) const;
<?php
$binaryOperators[] = '>';
?>
bool operator >(const <?php
echo $className;
?>
& other) const;
<?php
$binaryOperators[] = '<=';
?>
bool operator <=(const <?php
echo $className;
?>
& other) const;
<?php
$binaryOperators[] = '>=';
?>
bool operator >=(const <?php
echo $className;
?>
& other) const;
void FromString(const char* str);
int ToString(char* buffer) const;
void FromJson(const Json::Value& src);
void ToJson(Json::Value& dest) const;
};
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
()
: data() {
}
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
(const GrokitNull& null)
: data() {
}
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
(const char* str) {
this->FromString(str);
}
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
(const <?php
echo $className;
?>
::BitSet& data)
: data(data) {
}
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
(const <?php
echo $className;
?>
& other)
: data(other.data) {
}
inline bool <?php
echo $className;
?>
::IsEmpty() const {
return data == 0;
}
inline bool <?php
echo $className;
?>
::operator ==(const <?php
echo $className;
?>
& other) const {
return this->data == other.data;
}
inline bool <?php
echo $className;
?>
::operator !=(const <?php
echo $className;
?>
& other) const {
return this->data != other.data;
}
inline bool <?php
echo $className;
?>
::operator <(const <?php
echo $className;
?>
& other) const {
return this->data < other.data;
}
inline bool <?php
echo $className;
?>
::operator >(const <?php
echo $className;
?>
&other) const {
return this->data > other.data;
}
inline bool <?php
echo $className;
?>
::operator <=(const <?php
echo $className;
?>
& other) const {
return this->data <= other.data;
}
inline bool <?php
echo $className;
?>
::operator >=(const <?php
echo $className;
?>
& other) const {
return this->data >= other.data;
}
inline void <?php
echo $className;
?>
::FromString(const char* str) {
StorageType mask;
char* storage;
char* copy = strdup(str);
char* token = strtok_r(copy, " ", &storage);
while (token != NULL) {
Category level (token);
WARNINGIF(level.Invalid(), "Invalid token: %s", token);
mask |= 1 << level;
token = strtok_r(NULL, " ", &storage);
}
data = mask;
free(copy);
}
inline int <?php
echo $className;
?>
::ToString(char* buffer) const {
char* start = buffer;
<?php
foreach ($dictionary as $name) {
?>
if (data.<?php
echo $name;
?>
())
buffer += sprintf(buffer, "%s ", "<?php
echo $name;
?>
");
<?php
}
?>
if (start == buffer) {
buffer[0] = '\0';
return 1;
} else {
buffer[-1] = '\0';
return buffer - start;
}
}
inline void <?php
echo $className;
?>
::FromJson(const Json::Value& src) {
this->FromString(src.asCString());
}
inline void <?php
echo $className;
?>
::ToJson(Json::Value& dest) const {
char* buffer;
this->ToString(buffer);
dest = buffer;
}
<?php
ob_start();
?>
inline void FromString(@type& data, const char* buffer) {
data.FromString(buffer);
}
inline int ToString(const @type& data, char* buffer) {
return data.ToString(buffer);
}
inline void FromJson(const Json::Value& src, @type& dest) {
dest.FromJson(src);
}
inline void ToJson(const @type& src, Json::Value& dest) {
src.ToJson(dest);
}
<?php
$globalContent .= ob_get_clean();
?>
<?php
return ['kind' => 'TYPE', 'name' => $className, 'complex' => $complex, 'system_headers' => $systemHeaders, 'user_headers' => $userHeaders, 'lib_headers' => $libHeaders, 'libraries' => $libraries, 'binary_operators' => $binaryOperators, 'unary_operators' => $unaryOperators, 'global_content' => $globalContent, 'constructors' => $constructors, 'methods' => $methods, 'functions' => $functions, 'libraries' => $libraries, 'properties' => $properties, 'extras' => $extra, 'describe_json' => $describeJson];
}
function BASE64($t_args)
{
// The number of bytes encoded per string.
$size = get_first_key($t_args, ['size', 0]);
// The number of characters that should be used to encode each string.
// Padding is acceptable but not necessarily, hence the acceptable number of
// characters is an interval.
$min = ceil($size * 4 / 3);
$max = 4 * ceil($min / 4);
$className = 'Base64_' . $size;
$systemHeaders = ['cstring', 'cstdio'];
$userHeaders = [];
$libHeaders = ['base64.h'];
$libraries = [];
$constructors = [];
$methods = [];
$functions = [];
$binaryOperators = [];
$unaryOperators = [];
$globalContent = '';
$complex = false;
$properties = [];
$extra = ['size.bytes' => 40];
$describeJson = DescribeJson('base64', DescribeJsonStatic(['size' => $size]));
$globalContent = '';
?>
class <?php
echo $className;
?>
{
public:
// The number of bytes encoded.
static const constexpr int kSize = <?php
echo $size;
?>
;
// The maximum number of characters that an input string is expected to have.
static const constexpr int kMax = <?php
echo $max;
?>
;
// The minimum number of characters that an input string is expected to have.
static const constexpr int kMin = <?php
echo $min;
?>
;
private:
std::array<char, kSize> bytes;
public:
<?php
$constructors[] = [[], true];
?>
<?php
echo $className;
?>
();
<?php
$constructors[] = [['BASE::NULL'], true];
?>
<?php
echo $className;
?>
(const GrokitNull& nullval);
<?php
$constructors[] = [['BASE::STRING_LITERAL'], true];
?>
<?php
echo $className;
?>
(const char* str);
<?php
$binaryOperators[] = '==';
?>
bool operator ==(const <?php
echo $className;
?>
& other) const;
<?php
$binaryOperators[] = '!=';
?>
bool operator !=(const <?php
echo $className;
?>
& other) const;
void FromString(const char* str);
int ToString(char* buffer) const;
void FromJson(const Json::Value& src);
void ToJson(Json::Value& src) const;
};
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
() {
bytes.fill(0);
}
<?php
echo $className;
?>
::<?php
echo $className;
?>
(const GrokitNull& nullval) {
bytes.fill(0);
}
inline <?php
echo $className;
?>
::<?php
echo $className;
?>
(const char* str) {
this->FromString(str);
}
inline bool <?php
echo $className;
?>
::operator ==(const <?php
echo $className;
?>
& other) const {
return std::memcmp(bytes.data(), other.bytes.data(), kSize);
}
inline bool <?php
echo $className;
?>
::operator !=(const <?php
echo $className;
?>
& other) const {
return !(*this == other);
}
inline void <?php
echo $className;
?>
::FromString(const char* str) {
size_t length = strlen(str);
FATALIF(kMax < length || length < kMin,"Illegal base64-%d input: '%s'",
kSize, str);
std::string message = base64_decode(std::string(str));
std::memcpy(bytes.data(), message.data(), message.size());
}
inline int <?php
echo $className;
?>
::ToString(char* buffer) const {
std::string encoded = base64_encode((unsigned char*) bytes.data(), kSize);
return 1 + sprintf(buffer, "%s", encoded.c_str());
}
inline void <?php
echo $className;
?>
::FromJson(const Json::Value& src) {
this->FromString(src.asCString());
}
inline void <?php
echo $className;
?>
::ToJson(Json::Value& dest) const {
dest = base64_encode((unsigned char*) bytes.data(), kSize);
}
<?php
ob_start();
?>
inline void FromString(@type& data, const char* buffer) {
data.FromString(buffer);
}
inline int ToString(const @type& data, char* buffer) {
return data.ToString(buffer);
}
inline void FromJson(const Json::Value& src, @type& dest) {
dest.FromJson(src);
}
inline void ToJson(const @type& src, Json::Value& dest) {
src.ToJson(dest);
}
<?php
$globalContent .= ob_get_clean();
?>
<?php
return ['kind' => 'TYPE', 'name' => $className, 'complex' => $complex, 'system_headers' => $systemHeaders, 'user_headers' => $userHeaders, 'lib_headers' => $libHeaders, 'libraries' => $libraries, 'binary_operators' => $binaryOperators, 'unary_operators' => $unaryOperators, 'global_content' => $globalContent, 'constructors' => $constructors, 'methods' => $methods, 'functions' => $functions, 'libraries' => $libraries, 'properties' => $properties, 'extras' => $extra, 'describe_json' => $describeJson];
}
function Match($inputs, $args)
{
// Processing of template arguments.
$pattern = get_first_key($args, ['pattern', 0], 'Match: no pattern given.');
grokit_assert(is_string($pattern), 'Match: pattern should be a string.');
// Processing of inputs.
$count = \count($inputs);
grokit_assert($count == 1, "Match supports exactly 1 input. {$count} given");
$inputs_ = array_combine(['input'], $inputs);
$input = $inputs_['input']->name();
grokit_assert(in_array($input, ['BASE::STRING_LITERAL', 'BASE::STRING']), "Match: input should be a string (literal). {$input} given.");
$result = lookupType('BASE::BOOL');
$className = generate_name('Pattern');
$functName = generate_name('Matcher');
$userHeaders = ['PatternMatcherOnig.h'];
?>
class <?php
echo $className;
?>
{
public:
static PatternMatcherOnig pattern;
};
PatternMatcherOnig <?php
echo $className;
?>
::pattern("<?php
echo $pattern;
?>
");
<?php
echo $result;
?>
<?php
echo $functName;
?>
(<?php
echo const_typed_ref_args($inputs_);
?>
) {
<?php
if ($input == 'BASE::STRING_LITERAL') {
?>
return <?php
echo $className;
?>
::pattern.Match(input);
<?php
} else {
?>
return <?php
echo $className;
?>
::pattern.Match(input.ToString());
<?php
}
?>
}
<?php
return ['kind' => 'FUNCTION', 'name' => $functName, 'input' => $inputs, 'result' => $result, 'deterministic' => true, 'user_headers' => $userHeaders];
}
public function __construct($hash, $name, $value, array $args, array $origArgs)
{
parent::__construct(InfoKind::T_TYPE, $hash, $name, $value, $args, $origArgs[0]);
$iterator = 'ColumnIterator<' . $value . '>';
if (array_key_exists('complex', $args)) {
$iter = $args['complex'];
if ($iter !== false) {
$iterator = str_replace('@type', $this->value(), $iter);
}
}
$this->iterator = $iterator;
if (array_key_exists('destroy', $args)) {
$destroy = $args['destroy'];
if (is_bool($destroy)) {
$this->destroy = $destroy;
} else {
grokit_error("Expected boolean value for 'destroy' attribute of datatype " . $this->value());
}
}
if (array_key_exists('dictionary', $args)) {
$this->dictionary = $args['dictionary'];
}
if (array_key_exists('fixed_size', $args)) {
$this->fixedSize = $args['fixed_size'];
}
// Deal with operators defined by the type.
if (array_key_exists('unary_operators', $args)) {
foreach ($args['unary_operators'] as $op) {
LibraryManager::AlphaOperator($op, 1, $this);
}
}
if (array_key_exists('binary_operators', $args)) {
foreach ($args['binary_operators'] as $op) {
LibraryManager::AlphaOperator($op, 2, $this);
}
}
if (array_key_exists('constructors', $args)) {
foreach ($args['constructors'] as $cstr) {
$cArgs = get_first_key($cstr, ['args', 0]);
$cDet = get_first_key_default($cstr, ['deterministic', 1], true);
$myType = $this->value();
$myName = end(explode('::', $myType));
$cName = get_first_key_default($cstr, ['c_name', 2], $myName);
$callback = function ($args, $t_args = []) use($cArgs, $cDet, $myType, $cName) {
$myArgs = array_map('lookupType', $cArgs);
$myRet = lookupType($myType);
$retVal = ['kind' => 'FUNCTION', 'input' => $myArgs, 'result' => $myRet, 'deterministic' => $cDet];
if ($cName !== null) {
$retVal['name'] = $cName;
}
return $retVal;
};
declareFunctionGlobal('', $myType, $cArgs, $callback);
}
}
if (array_key_exists('functions', $args)) {
foreach ($args['functions'] as $fnct) {
$fName = get_first_key($fnct, ['name', 0]);
$fArgs = get_first_key($fnct, ['args', 1]);
$fRet = get_first_key($fnct, ['result', 2]);
$fDet = get_first_key_default($fnct, ['deterministic', 3], false);
$fGlobal = get_first_key_default($fnct, ['global', 4], false);
$cName = get_first_key_default($fnct, ['c_name', 5], null);
foreach ($fArgs as &$arg) {
$arg = str_replace('@type', $this->value(), $arg);
}
$fRet = str_replace('@type', $this->value(), $fRet);
$callback = function ($args, $t_args = []) use($fArgs, $fRet, $fDet, $cName, $fGlobal) {
$myArgs = array_map('lookupType', $fArgs);
$myRet = lookupType($fRet);
$retVal = ['kind' => 'FUNCTION', 'input' => $myArgs, 'result' => $myRet, 'deterministic' => $fDet, 'global' => $fGlobal];
if ($cName !== null) {
$retVal['name'] = $cName;
}
return $retVal;
};
$fNS = $fGlobal ? 'BASE' : LibraryManager::ExtractNamespace($this->value());
$fName = LibraryManager::JoinNamespace($fNS, $fName);
declareFunctionGlobal('', $fName, $fArgs, $callback);
}
}
if (array_key_exists('methods', $args)) {
foreach ($args['methods'] as $method) {
$mName = get_first_key($method, ['name', 0]);
$mArgs = get_first_key($method, ['args', 1]);
$mRet = get_first_key($method, ['result', 2]);
$mDet = get_first_key_default($method, ['deterministic', 3], false);
if ($mRet == '@type') {
$mRet = $this->value();
}
foreach ($mArgs as &$mArg) {
if ($mArg == '@type') {
$mArg = $this->value();
}
}
//fwrite(STDERR, 'Defining method ' . $this->value() . '->' . $mName .
//'(' . implode(', ', $mArgs ) . ') : ' . $mRet . PHP_EOL );
$info = new MethodInfo($mName, $mArgs, $mRet, $mDet);
if (!array_key_exists($mName, $this->methods)) {
$this->methods[$mName] = [$info];
} else {
foreach ($this->methods[$mName] as $cm) {
if ($info == $cm) {
grokit_error('Attempting to multiply define method ' . $this->value() . '->' . $mName . '(' . implode(', ', $mArgs) . ')');
}
}
$this->methods[$mName][] = $info;
}
}
}
if (array_key_exists('describe_json', $args)) {
$this->describers['json'] = $args['describe_json'];
}
}
/**
* generate title, description and source link of a feed
*/
function generate_feed_details($content, $global, &$feed_data)
{
global $board_url, $config, $user, $db, $phpEx;
if ($content == 'pm') {
global $folder;
// custom folder?
if (is_int($folder)) {
global $folder_name;
} else {
$user->add_lang('ucp');
$folder_name = $user->lang['PM_' . strtoupper($folder)];
}
$title = sprintf($user->lang['SYNDICATION_PM_TITLE'], $folder_name);
$description = sprintf($user->lang['SYNDICATION_PM_DESCRIPTION'], $folder_name, $config['sitename']);
$source_link = "ucp.{$phpEx}?i=pm&folder={$folder}";
} else {
// give the feed a name and description
if ($global) {
$title = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_GLOBAL_TITLE'], $config['sitename']);
$description = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_GLOBAL_DESCRIPTION'], $config['sitename']);
$source_link = "index.{$phpEx}";
} else {
if ($content == 'topic_posts') {
global $topic_row, $topic_id;
$title = sprintf($user->lang['SYNDICATION_TOPIC_POSTS_TITLE'], $topic_row['topic_title']);
$description = sprintf($user->lang['SYNDICATION_TOPIC_POSTS_DESCRIPTION'], $topic_row['topic_title'], $config['sitename']);
$source_link = "viewtopic.{$phpEx}?f={$topic_row['forum_id']}&t={$topic_id}";
} else {
global $single_cat, $only_cats, $forum_ids;
if ($single_cat) {
global $forum_name;
$title = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_CATEGORY_TITLE'], $forum_name);
$description = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_CATEGORY_DESCRIPTION'], $forum_name, $config['sitename']);
$source_link = "viewforum.{$phpEx}?f=" . get_first_key($forum_ids);
} else {
if ($only_cats) {
$title = $user->lang['SYNDICATION_' . strtoupper($content) . '_CATEGORIES_TITLE'];
$description = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_CATEGORIES_DESCRIPTION'], $config['sitename']);
$source_link = "index.{$phpEx}";
} else {
// if we have only a single forum, get details about it
if (sizeof($forum_ids) == 1) {
$sql = 'SELECT forum_name
FROM ' . FORUMS_TABLE . '
WHERE forum_id = ' . get_first_key($forum_ids);
$result = $db->sql_query($sql, 3600);
$forum_name = $db->sql_fetchfield('forum_name', 0, $result);
$db->sql_freeresult($result);
$title = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_TITLE'], $forum_name);
$description = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_DESCRIPTION'], $forum_name, $config['sitename']);
$source_link = "viewforum.{$phpEx}?f=" . get_first_key($forum_ids);
} else {
$title = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_VARIOUS_TITLE'], $config['sitename']);
$description = sprintf($user->lang['SYNDICATION_' . strtoupper($content) . '_VARIOUS_DESCRIPTION'], $config['sitename']);
$source_link = "index.{$phpEx}";
}
}
}
}
}
}
$feed_data += array('title' => $title, 'description' => $description, 'source_link' => "{$board_url}/{$source_link}", 'lang' => $user->data['user_lang']);
}
function CSVReader(array $t_args, array $output)
{
$my_output = [];
// Handle case where outputs are given as template arguments
// and not implied.
if (\count($output) == 0) {
grokit_assert(array_key_exists('output', $t_args), 'Did not receive any description of my output!');
$output_list = $t_args['output'];
grokit_assert(is_array($output_list), 'Expected list of types for template argument "output"');
$i = 1;
foreach ($outputs_list as $name => $out_type) {
grokit_assert(is_datatype($out_type) || is_identifier($out_type), 'Expected only types in the "output" list');
if (is_identifier($out_type)) {
$out_type = lookupType($out_type->value());
}
$name = 'val_' . $i;
$my_output[$name] = $out_type;
$i += 1;
}
} else {
foreach ($output as $key => $out) {
$name = $key;
$my_output[$name] = $out;
}
}
$debug = get_default($t_args, 'debug', 0);
$simple = get_default($t_args, 'simple', false);
$trimCR = get_default($t_args, 'trim.cr', false);
// Handle separator
$separator = ',';
if (array_key_exists('sep', $t_args) || array_key_exists('separator', $t_args)) {
$sep = get_first_key($t_args, ['sep', 'separator']);
grokit_assert(is_string($sep), "Got " . gettype($sep) . " instead of string for separator.");
if (strtolower($sep) === 'tab') {
$sep = '\\t';
}
grokit_assert($sep != "\n", 'CSV column delimiter cannot be new line');
// Scream if separator is longer than one character
grokit_assert(\strlen($sep) == 1 || $sep == '\\t', 'Expected string of length 1 for separator, got string <' . $sep . '> instead');
$separator = $sep;
}
// Handle quote character
$quotechar = '"';
if (array_key_exists('quote', $t_args) && !is_null($t_args['quote'])) {
grokit_assert(!$simple, 'Quote option not available for simple CSVReader');
$quote = $t_args['quote'];
grokit_assert(is_string($quote), "Got " . gettype($quote) . " instead of string for quote.");
// Scream if separator is longer than one character
grokit_assert(\strlen($quote) == 1, 'Expected string of length 1 for quote character, got string <' . $quote . '> instead');
$quotechar = $quote;
}
$quotechar = addcslashes($quotechar, '\\\'');
// Handle escape character
$escapeChar = '\\';
if (array_key_exists('escape', $t_args) && !is_null($t_args['escape'])) {
grokit_assert(!$simple, 'Escape option not available for simple CSVReader');
$escape = $t_args['escape'];
grokit_assert(is_string($escape), 'Got ' . gettype($escape) . ' instead of string for escape character.');
grokit_assert(\strlen($escape) == 1, 'Expected string of length 1 for escape character, got string <' . $escape . '> instead');
$escapeChar = $escape;
}
$escapeChar = addcslashes($escapeChar, '\\\'');
// Handle header lines
$headerLines = 0;
if (array_key_exists('skip', $t_args)) {
$headerLines = $t_args['skip'];
grokit_assert(is_int($headerLines), 'Got ' . gettype($headerLines) . ' instead of int for number of lines to skip.');
grokit_assert($headerLines >= 0, 'Cannot skip a negative number of lines.');
}
// Maximum number of lines to read
$maxLines = get_default($t_args, 'n', -1);
grokit_assert(is_int($maxLines), 'Got ' . gettype($maxLines) . ' instead of int for template argument "n"');
$nullArg = get_first_key_default($t_args, ['nullable'], false);
$nullable = [];
$nullStr = [];
foreach ($my_output as $name => $type) {
$nullable[$name] = false;
}
if ($nullArg === true) {
foreach ($my_output as $name => $type) {
$nullable[$name] = true;
$nullStr[$name] = 'NULL';
}
} else {
if (is_array($nullArg)) {
foreach ($nullArg as $n => $v) {
// If nullable value is an associative mapping, the value is either true/false
// or the value of the null string
if (is_string($n)) {
grokit_assert(is_string($v) || is_bool($v), 'CSVReader: nullable associative mapping must have string or boolean values');
grokit_assert(array_key_exists($n, $nullable), 'CSVReader: cannot make unknown attribute ' . $n . ' nullable');
if (is_bool($v)) {
$nullable[$n] = $v;
$nullStr[$n] = 'NULL';
} else {
$nullable[$n] = true;
$nullStr[$n] = $v;
}
} else {
if (is_array($v)) {
grokit_assert(array_key_exists('attr', $v), 'CSVReader: Name of nullable attribute not specified');
$attrName = $v['attr']->name();
$nullable[$attrName] = true;
$nullStr[$attrName] = array_key_exists('null', $v) ? $v['null'] : 'NULL';
} else {
// Otherwise, it's just nullable
$attrName = $v->name();
grokit_assert(array_key_exists($attrName, $nullable), 'CSVReader: cannot make unknown attribute ' . $v . ' nullable');
$nullable[$attrName] = true;
$nullStr[$attrName] = 'NULL';
}
}
}
} else {
if ($nullArg === false) {
// Nothing
} else {
if (is_string($nullArg)) {
foreach ($my_output as $name => $type) {
$nullable[$name] = true;
$nullStr[$name] = $nullArg;
}
} else {
grokit_error('Template argument "nullable" must be boolean or array, ' . typeof($nullArg) . ' given');
}
}
}
}
// Come up with a name for ourselves
$className = generate_name('CSVReader');
if ($debug >= 2) {
foreach ($my_output as $name => $type) {
fwrite(STDERR, "CSVReader: {$name} is nullable: " . ($nullable[$name] ? 'true' : 'false') . PHP_EOL);
}
}
?>
class <?php
echo $className;
?>
{
std::istream& my_stream;
std::string fileName;
// Template parameters
static constexpr size_t MAX_LINES = <?php
echo $maxLines;
?>
;
static constexpr size_t HEADER_LINES = <?php
echo $headerLines;
?>
;
static constexpr char DELIMITER = '<?php
echo $separator;
?>
';
<?php
if (!$simple) {
?>
static constexpr char QUOTE_CHAR = '<?php
echo $quotechar;
?>
';
static constexpr char ESCAPE_CHAR = '<?php
echo $escapeChar;
?>
';
typedef boost::escaped_list_separator<char> separator;
typedef boost::tokenizer< separator > Tokenizer;
separator my_separator;
Tokenizer my_tokenizer;
<?php
}
?>
// Prevent having to allocate this every time.
std::string line;
std::vector<std::string> tokens;
size_t count;
<?php
\grokit\declareDictionaries($my_output);
?>
public:
<?php
echo $className;
?>
( GIStreamProxy& _stream ) :
my_stream(_stream.get_stream())
, fileName(_stream.get_file_name())
<?php
if (!$simple) {
?>
, my_separator(ESCAPE_CHAR, DELIMITER, QUOTE_CHAR)
, my_tokenizer(std::string(""))
<?php
}
?>
, count(0)
{
<?php
if ($headerLines > 0) {
?>
for( size_t i = 0; i < HEADER_LINES; ++i ) {
FATALIF( !getline( my_stream, line ), "CSV Reader reached end of file before finishing header.\n" );
}
<?php
}
// If headerLines > 0
?>
}
// >
bool ProduceTuple( <?php
echo typed_ref_args($my_output);
?>
) {
if (count < MAX_LINES) { //>
count++;
} else {
return false;
}
if( getline( my_stream, line ) ) {
<?php
if ($trimCR) {
?>
if( line.back() == '\r' ) {
line.pop_back();
}
<?php
}
// if trimCR
if (!$simple) {
if ($debug >= 1) {
?>
try {
<?php
}
// if debug >= 1
?>
my_tokenizer.assign( line, my_separator );
<?php
if ($debug >= 1) {
?>
} catch(...) {
FATAL("CSVReader for file %s failed on line: %s", fileName.c_str(), line.c_str());
}
<?php
}
// if debug >= 1
?>
Tokenizer::iterator it = my_tokenizer.begin();
<?php
foreach ($my_output as $name => $type) {
if ($nullable[$name]) {
// nullable
?>
<?php
\grokit\fromStringNullable($name, $type, 'it->c_str()', true, $nullStr[$name]);
?>
<?php
} else {
// not nullable
?>
<?php
echo \grokit\fromStringDict($name, $type, 'it->c_str()');
?>
;
<?php
}
// end nullable check
?>
++it;
<?php
}
// foreach output
} else {
?>
for( char & c : line ) {
if( c == DELIMITER )
c = '\0';
}
const char * ptr = line.c_str();
<?php
$first = true;
foreach ($my_output as $name => $type) {
if ($first) {
$first = false;
} else {
?>
while( *(ptr++) != '\0' )
; // Advance past next delimiter
<?php
}
// not first output
if ($nullable[$name]) {
?>
<?php
echo \grokit\fromStringNullable($name, $type, 'ptr', true, $nullStr[$name]);
} else {
// not nullable
?>
<?php
echo \grokit\fromStringDict($name, $type, 'ptr');
?>
;
<?php
}
// if nullable
}
// foreach output
}
// if simple reader
?>
return true;
}
else {
return false;
}
}
<?php
\grokit\declareDictionaryGetters($my_output);
?>
};
<?php
$sys_headers = ['vector', 'string', 'iostream', 'cstdint'];
if (!$simple) {
$sys_headers[] = 'boost/tokenizer.hpp';
}
return ['name' => $className, 'kind' => 'GI', 'output' => $my_output, 'system_headers' => $sys_headers, 'user_headers' => ['GIStreamInfo.h', 'Dictionary.h', 'DictionaryManager.h']];
}