/** * GI that generates data in clusters, using a specified distribution for each * cluster. * * This GI requires the following template arguments: * - 'n' or 0 * The number of tuples to generate. Note: this value is per task. * The total number of tuples generated will be n_tasks * n * - 'centers' or 1 * A list of configuration for the centers. * * The following template arguments are optional: * - 'outputs' * If the outputs of the GI are not given implicitly, they can be * specified in this template argument. The number of dimensions will * be determined by the number of outputs. * * All output types must be numeric real types. The default type for * outputs is DOUBLE. * - 'dist.lib' = 'std' * Which library to use for generating distributions. * Valid options are: * - std * - boost * - 'seed' = null * The seed to be used for the random number generator. This seed will * be used to generate the seed for each task, and different runs with * the same seed will produce the same data. * - 'compute.sets' = 1 * The number of sets of tuples to compute at once. * * Each center configuration is a functor with the form: * dist_name(args) * * The following distributions are supported: * { Uniform Distributions } * - uniform(a = 0, b = 1) * * { Normal Distributions } * - normal(mean = 0.0, std_dev = 1.0) [ synonyms: gaussian ] * - inverse_gaussian(mean = 1, shape = 1) [ synonyms: inverse_normal ] * * { Bernoulli Distributions } * - binomial(t = 1, p = 0.5) * - negative_binomial(k = 1, p = 0.5) * * { Poisson Distributions } * - exponential( lambda = 1 ) * - gamma(alpha = 1, beta = 1) [ synonyms: Gamma ] */ function ClusterGen(array $t_args, array $outputs) { $sys_headers = ['array', 'cinttypes']; $user_headers = []; $libraries = []; if (\count($outputs) == 0) { grokit_assert(array_key_exists('outputs', $t_args), 'ClusterGen: No outputs specified'); $count = 0; foreach ($t_args['outputs'] as $type) { if (is_identifier($type)) { $type = lookupType($type); } grokit_assert(is_datatype($type), 'ClusterGen: Non data-type ' . $type . ' given as output'); $name = 'output' . $count++; $outputs[$name] = $type; } } foreach ($outputs as $name => &$type) { if (is_null($type)) { $type = lookupType('base::DOUBLE'); } else { grokit_assert($type->is('real'), 'ClusterGen: Non-real datatype ' . $type . ' given as output'); } } $myOutputs = []; foreach ($outputs as $name => $type) { $myOutputs[$name] = $type; } $tSize = \count($outputs); $seed = get_default($t_args, 'seed', null); if ($seed !== null) { grokit_assert(is_int($seed), 'ClusterGen: Seed must be an integer or null.'); } else { $user_headers[] = 'HashFunctions.h'; } $distLib = get_default($t_args, 'dist.lib', 'std'); $distNS = ''; switch ($distLib) { case 'std': $sys_headers[] = 'random'; $distNS = 'std'; break; case 'boost': $sys_headers[] = 'boost/random.hpp'; $distNS = 'boost::random'; $libraries[] = 'boost_random-mt'; if ($seed === null) { // Need random_device $sys_headers[] = 'boost/random/random_device.hpp'; $libraries[] = 'boost_system-mt'; } break; default: grokit_error('ClusterGen: Unknown RNG library ' . $distLib); } $distRNG = 'mt19937'; $RNGtype = $distNS . '::' . $distRNG; $nTuples = get_first_key($t_args, ['n', '0']); grokit_assert(is_int($nTuples), 'ClusterGen: the number of tuples to be produced must be an integer.'); $centers = get_first_key($t_args, ['centers', 1]); grokit_assert(is_array($centers), 'ClusterGen: centers must be an array of functors'); $handleDist = function ($name, $args, $oType) use($distNS) { $distName = ''; $distArgs = []; switch ($name) { case 'gaussian': case 'normal': $distName = $distNS . '::' . 'normal_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Normal distribution takes at most 2 arguments, ' . \count($args) . ' given'); $mean = get_default($args, ['mean', 0], 0.0); $sigma = get_default($args, ['std_dev', 'sigma', 1], 1.0); grokit_assert(is_numeric($mean), 'ClusterGen: mean parameter of binomial distribution must be a real number.'); grokit_assert(is_numeric($sigma), 'ClusterGen: sigma parameter of binomial distribution must be a real number.'); $mean = floatval($mean); $sigma = floatval($sigma); $distArgs = [$mean, $sigma]; break; case 'binomial': $distName = $distNS . '::' . 'binomial_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Binomial distribution takes at most 2 arguments, ' . \count($args) . ' given'); $t = get_default($args, ['t', 0], 1); $p = get_default($args, ['p', 1], 0.5); grokit_assert(is_int($t), 'ClusterGen: t parameter of binomial distribution must be an integer.'); grokit_assert(is_numeric($p), 'ClusterGen: p parameter of binomial distribution must be a real number.'); $p = floatval($p); grokit_assert($p >= 0 && $p <= 1, 'ClusterGen: p parameter of binomial distribution must be in the range [0, 1]'); grokit_assert($t >= 0, 'ClusterGen: t parameter of binomial distribution must be in the range [0, +inf)'); $distArgs = [$t, $p]; break; case 'negative_binomial': $distName = $distNS . '::' . 'negative_binomial_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Negative Binomial distribution takes at most 2 arguments, ' . \count($args) . ' given'); $k = get_default($args, ['k', 0], 1); $p = get_default($args, ['p', 1], 0.5); grokit_assert(is_int($k), 'ClusterGen: k parameter of binomial distribution must be an integer.'); grokit_assert(is_numeric($p), 'ClusterGen: p parameter of binomial distribution must be a real number.'); $p = floatval($p); grokit_assert($p > 0 && $p <= 1, 'ClusterGen: p parameter of negative binomial distribution must be in the range (0, 1]'); grokit_assert($k > 0, 'ClusterGen: k parameter of negative binomial distribution must be in the range (0, +inf)'); $distArgs = [$k, $p]; break; case 'inverse_gaussian': case 'inverse_normal': grokit_assert(\count($args) <= 2, 'ClusterGen: Inverse Gaussian distribution takes at most 2 arguments, ' . \count($args) . ' given'); $mean = get_default($args, ['mean', 0], 1); $shape = get_default($args, ['shape', 1], 1); grokit_assert(is_numeric($mean), 'ClusterGen: mean parameter of inverse gaussian distribution must be a real number.'); grokit_assert(is_numeric($shape), 'ClusterGen: shape parameter of inverse gaussian distribution must be a real number.'); $mean = floatval($mean); $shape = floatval($shape); grokit_assert($mean > 0, 'ClusterGen: mean of inverse gaussian distribution must be in range (0, inf)'); grokit_assert($shape > 0, 'ClusterGen: shape of inverse gaussian distribution must be in range (0, inf)'); $gen_args = ['output' => $oType, 'ns' => $distNS]; $distName = strval(lookupResource('datagen::InverseGaussianGen', $gen_args)); $distArgs = [$mean, $shape]; break; case 'uniform': $distName = $distNS . '::' . 'uniform_real_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Uniform distribution takes at most 2 arguments, ' . \count($args) . ' given'); $a = get_default($args, ['a', 0], 0.0); $b = get_default($args, ['b', 1], 1.0); grokit_assert(is_numeric($a), 'ClusterGen: `a` parameter of uniform distribution must be a real number.'); grokit_assert(is_numeric($b), 'ClusterGen: `b` parameter of uniform distribution must be a real number.'); $a = floatval($a); $b = floatval($b); grokit_assert($b >= $a, 'ClusterGen: `b` parameter of uniform distribution must be >= the `a` parameter.'); $distArgs = [$a, $b]; break; case 'exponential': $distName = $distNS . '::' . 'exponential_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 1, 'ClusterGen: Exponential distribution takes at most 1 argument.'); $lambda = get_default($args, ['lambda', 0], 1.0); grokit_assert(is_numeric($lambda), 'ClusterGen: `lambda` parameter of exponential distribution must be a real number.'); $lambda = floatval($lambda); grokit_assert($lambda > 0, 'ClusterGen: `lambda` parameter of exponential distribution must be in range (0, +inf).'); $distArgs = [$lambda]; break; case 'gamma': case 'Gamma': $distName = $distNS . '::' . 'gamma_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Gamma distribution takes at most 2 arguments.'); $alpha = get_default($args, ['alpha', 0], 1.0); $beta = det_default($args, ['beta', 1], 1.0); grokit_assert(is_numeric($alpha), 'ClusterGen: `alpha` parameter of gamma distribution must be a real number.'); grokit_assert(is_numeric($beta), 'ClusterGen: `beta` parameter of gamma distribution must be a real number.'); $alpha = floatval($alpha); $beta = floatval($beta); $distArgs = [$alpha, $beta]; break; default: grokit_error('ClusterGen: Unknown distribution ' . $name . ' given for center'); } return [$distName, $distArgs]; }; $dists = []; $distArgs = []; $count = 0; $oType = ''; $nCenters = 1; reset($outputs); foreach ($centers as $val) { $cluster = $val; if (is_functor($val)) { $cluster = [$val]; } else { if (is_array($val)) { $nCenters = lcm($nCenters, \count($val)); } else { grokit_error('ClusterGen: center descriptions must be functors or list of functors'); } } $curDist = []; $curDistArgs = []; $curDistName = 'distribution' . $count++; $oType = strval(current($outputs)); $iCount = 0; foreach ($cluster as $functor) { grokit_assert(is_functor($functor), 'ClusterGen: center description must be a functor'); $vName = $curDistName . '_' . $iCount++; $ret = $handleDist($functor->name(), $functor->args(), $oType); $curDist[$vName] = $ret[0]; $curDistArgs[$vName] = $ret[1]; } next($outputs); $dists[$curDistName] = $curDist; $distArgs[$curDistName] = $curDistArgs; } // Determine the default number of sets to compute at a time. // We want to generate either $nTuples or 10,000 tuples, depending on which // is less. $defaultSetsTarget = min($nTuples, 10000); $setsToTarget = intval(ceil($defaultSetsTarget / $nCenters)); $computeSets = get_default($t_args, 'compute.sets', $setsToTarget); grokit_assert(is_int($computeSets) && $computeSets > 0, 'ClusterGen: compute.sets must be a positive integer, ' . $computeSets . ' given'); $className = generate_name('ClusterGen'); // For some BIZZARE reason, the $outputs array was getting modified while // traversing over the $dists array. Making a deep copy of the outputs and // then reassigning it seems to fix the issue. $outputs = $myOutputs; ?> class <?php echo $className; ?> { // The number of tuples to produce per task static constexpr size_t N = <?php echo $nTuples; ?> ; static constexpr size_t CacheSize = <?php echo $computeSets * $nCenters; ?> ; // Typedefs typedef std::tuple<<?php echo array_template('{val}', ', ', $outputs); ?> > Tuple; typedef std::array<Tuple, CacheSize> TupleArray; typedef TupleArray::const_iterator TupleIterator; typedef <?php echo $RNGtype; ?> RandGen; // Number of tuples produced. uintmax_t count; // Cache a number of outputs for efficiency TupleArray cache; TupleIterator cacheIt; // Random number generator RandGen rng; // Distributions <?php // This is the section causing issues. foreach ($dists as $name => $list) { foreach ($list as $vName => $type) { ?> <?php echo $type; ?> <?php echo $vName; ?> ; <?php } // foreach distribution } // foreach cluster set ?> // Helper function to generate tuples. void GenerateTuples(void) { <?php $tIndex = 0; foreach ($dists as $name => $list) { $lCenters = \count($list); // $nCenters has been defined to be the LCM of the number of centers in // any column, so $lCenter is guaranteed to divide evenly into // CacheSize ?> for( size_t index = 0; CacheSize > index; index += <?php echo $lCenters; ?> ) { <?php $index = 0; foreach ($list as $vName => $type) { ?> std::get<<?php echo $tIndex; ?> >(cache[index + <?php echo $index; ?> ]) = <?php echo $vName; ?> (rng); <?php $index++; } // foreach value in tuple ?> } <?php $tIndex++; } // foreach distribution ?> cacheIt = cache.cbegin(); } public: // Constructor <?php echo $className; ?> ( GIStreamProxy & _stream ) : cache() , cacheIt() , count(0) , rng() <?php foreach ($dists as $name => $list) { foreach ($list as $vName => $type) { ?> , <?php echo $vName; ?> (<?php echo implode(', ', $distArgs[$name][$vName]); ?> ) <?php } // foreach distribution } // foreach cluster set ?> { <?php if (is_null($seed)) { ?> <?php echo $distNS; ?> ::random_device rd; <?php } // if seed is null ?> RandGen::result_type seed = <?php echo is_null($seed) ? 'rd()' : "CongruentHash({$seed}, _stream.get_id() )"; ?> ; rng.seed(seed); cacheIt = cache.cend(); } // Destructor ~<?php echo $className; ?> (void) { } bool ProduceTuple(<?php echo typed_ref_args($outputs); ?> ) { if( N > count ) { if( cacheIt == cache.cend() ) { GenerateTuples(); } <?php $tIndex = 0; foreach ($outputs as $name => $type) { ?> <?php echo $name; ?> = std::get<<?php echo $tIndex; ?> >(*cacheIt); <?php $tIndex++; } // foreach output ?> ++cacheIt; ++count; return true; } else { return false; } } }; <?php return array('kind' => 'GI', 'name' => $className, 'output' => $outputs, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'libraries' => $libraries); }
function ExtremeTuples(array $t_args, array $inputs, array $outputs) { $extremes = get_first_key($t_args, ['extremes']); $nExt = \count($extremes); grokit_assert($nExt > 0, 'No extremes specified for ExtremeTuples GLA.'); if (\count($inputs) == 0) { grokit_assert(array_key_exists('inputs', $t_args), 'No arguments specified for ExtremeTuples GLA.'); $count = 0; foreach ($t_args['inputs'] as $type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Only datatypes can be specified as inputs to ' . 'the ExtremeTuples GLA'); $name = 'et_val' . $count; $inputs[$name] = $type; } } $outputMap = []; reset($outputs); foreach ($inputs as $name => $type) { $oKey = key($outputs); $outputs[$oKey] = $type; $outputMap[$oKey] = $name; next($outputs); } grokit_assert($nExt <= \count($inputs), 'There can not be more extreme values than there are inputs!'); $mainAtts = []; $extraAtts = []; $minOpts = ['MIN', 'MINIMUM', '-', '<']; $maxOpts = ['MAX', 'MAXIMUM', '+', '>']; $inArrayCase = function ($needle, $haystack) { foreach ($haystack as $item) { if (strcasecmp($needle, $item) == 0) { return true; } } return false; }; $minimum = []; foreach ($extremes as $name => $val) { grokit_assert(array_key_exists($name, $inputs), "ExtremeTuples: Expression with name " . $name . " specified as extreme not found in inputs"); } foreach ($inputs as $name => $type) { if (array_key_exists($name, $extremes)) { $mainAtts[$name] = $type; if ($inArrayCase($extremes[$name], $minOpts)) { $minimum[$name] = true; } else { if ($inArrayCase($extremes[$name], $maxOpts)) { $minimum[$name] = false; } else { grokit_error('Unknown extreme type ' . $extremes[$name] . ' specified for ' . $name); } } } else { $extraAtts[$name] = $type; } } $debug = get_default($t_args, 'debug', 0); $className = generate_name('ExtremeTuples'); ?> class <?php echo $className; ?> { struct Tuple { <?php foreach ($inputs as $name => $type) { ?> <?php echo $type; ?> <?php echo $name; ?> ; <?php } // foreach input ?> // Default Constructor, Copy Constructor, and Copy Assignment are all // default Tuple(void) = default; Tuple(const Tuple &) = default; Tuple & operator = (const Tuple &) = default; Tuple(<?php echo array_template('const {val} & _{key}', ', ', $inputs); ?> ) : <?php echo array_template('{key}(_{key})', ', ', $inputs); ?> { } // operator > means that this tuple is "better" than the other tuple. bool operator > ( const Tuple & other ) const { <?php foreach ($mainAtts as $name => $type) { $op1 = $minimum[$name] ? '<' : '>'; $op2 = !$minimum[$name] ? '<' : '>'; ?> if( <?php echo $name; ?> <?php echo $op1; ?> other.<?php echo $name; ?> ) return true; else if( <?php echo $name; ?> <?php echo $op2; ?> other.<?php echo $name; ?> ) return false; <?php } // foreach main attribute ?> return false; } bool operator < ( const Tuple& other ) const { return other > *this; } bool operator <= (const Tuple & other ) const { return ! (*this > other ); } bool operator >= (const Tuple & other ) const { return !( other > *this ); } bool operator == (const Tuple & other ) const { bool ret = true; <?php foreach ($mainAtts as $name => $type) { ?> ret &= <?php echo $name; ?> == other.<?php echo $name; ?> ; <?php } // foreach main attribute ?> return ret; } }; // struct Tuple typedef std::vector<Tuple> TupleVector; public: class Iterator { public: typedef TupleVector::const_iterator iter_type; private: iter_type begin; iter_type end; public: Iterator(void) = default; Iterator(const Iterator &) = default; Iterator( const iter_type & _begin, const iter_type & _end ) : begin(_begin), end(_end) { } Iterator( const iter_type && _begin, const iter_type && _end ) : begin(_begin), end(_end) { } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { if( begin != end ) { <?php foreach ($outputs as $name => $type) { ?> <?php echo $name; ?> = begin-><?php echo $outputMap[$name]; ?> ; <?php } ?> begin++; return true; } else { return false; } } }; private: uintmax_t __count; // number of tuples covered TupleVector tuples; // Iterator for multi output type Iterator multiIterator; public: // Constructor and destructor <?php echo $className; ?> (void) : __count(0), tuples(), multiIterator() { } ~<?php echo $className; ?> () { } void AddItem( <?php echo const_typed_ref_args($inputs); ?> ) { ++__count; Tuple t(<?php echo args($inputs); ?> ); if( tuples.empty() ) { tuples.push_back(t); } else if( t > tuples.front() ) { tuples.clear(); tuples.push_back(t); } else if( t == tuples.front() ) { tuples.push_back(t); } } void AddState( <?php echo $className; ?> & other ) { if( tuples.size() == 0 ) { tuples.swap(other.tuples); } else if( other.tuples.size() == 0 ) { // Do nothing } else if( tuples.front() > other.tuples.front() ) { // fast path } else if( other.tuples.front() > tuples.front() ) { tuples.swap(other.tuples); } else { for( Tuple & t : other.tuples ) { tuples.push_back(t); } } } void Finalize( void ) { multiIterator = Iterator(tuples.cbegin(), tuples.cend()); } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { return multiIterator.GetNextResult(<?php echo args($outputs); ?> ); } }; // class <?php echo $className; ?> <?php $system_headers = ['vector', 'algorithm', 'cinttypes']; if ($debug > 0) { $system_headers = array_merge($system_headers, ['iostream', 'sstream', 'string']); } return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => 'multi', 'system_headers' => $system_headers); }
function GroupBy(array $t_args, array $inputs, array $outputs, array $states) { // Ensure we have valid inputs. if (\count($inputs) == 0) { // No inputs given, try to get them from template arguments. grokit_assert(array_key_exists('input', $t_args), 'No inputs given for GroupBy'); $inputs = $t_args['input']; if (!is_array($inputs)) { $inputs = [$inputs]; } foreach ($inputs as $name => &$type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Invalid type given for input ' . $name); } } grokit_assert(array_key_exists('group', $t_args), 'No groups specified for GroupBy'); $gbyAttMap = $t_args['group']; grokit_assert(is_array($gbyAttMap), 'Invalid value given for groups, expected an expression name or list of expression names'); $gbyAttMap = array_map('strval', $gbyAttMap); $gbyAttNames = array_keys($gbyAttMap); foreach ($gbyAttMap as $in => $out) { grokit_assert(array_key_exists($in, $inputs), 'Group ' . $in . ' not present in input'); grokit_assert(array_key_exists($out, $outputs), 'Output Attribute ' . $out . ' for group ' . $in . ' not found in outputs'); } $numGByAtts = \count($gbyAttNames); grokit_assert(array_key_exists('aggregate', $t_args), 'No aggregate specified for GroupBy'); $innerGLA = $t_args['aggregate']; grokit_assert(is_gla($innerGLA), 'Non-GLA specified as aggregate for GroupBy'); $debug = get_default($t_args, 'debug', 0); $init_size = get_default($t_args, 'init.size', 1024); $use_mct = get_default($t_args, 'use.mct', true); $keepHashes = get_default($t_args, 'mct.keep.hashes', false); grokit_assert(is_bool($keepHashes), 'GroupBy mct.keep.hashes argument must be boolean'); // determine the result type $use_fragments = get_default($t_args, 'use.fragments', true); $resType = $use_fragments ? ['fragment', 'multi'] : ['multi']; $fragSize = get_default($t_args, 'fragment.size', 2000000); // Always support state $resType[] = 'state'; // Class name randomly generated $className = generate_name("GroupBy"); // instantiate the inner GLA. input/output is derived from the main input/output $gbyAtts = []; $gbyAttsOut = []; $glaInputAtts = []; $glaOutputAtts = []; foreach ($inputs as $name => $type) { if (in_array($name, $gbyAttNames)) { $gbyAtts[$name] = $type; $gbyAttsOut[$gbyAttMap[$name]] = $type; $outputs[$gbyAttMap[$name]] = $type; } else { $glaInputAtts[$name] = $type; } } foreach ($outputs as $name => $type) { if (!in_array($name, $gbyAttMap)) { $glaOutputAtts[$name] = $type; } } $innerGLA = $innerGLA->apply($glaInputAtts, $glaOutputAtts, $states); $libraries = $innerGLA->libraries(); $innerRes = get_first_value($innerGLA->result_type(), ['multi', 'single', 'state']); if ($innerRes == 'state') { // If the result type is state, the only output is a state object // containing the GLA. $outputName = array_keys($glaOutputAtts)[0]; $innerOutputs = [$outputName => lookupType('base::STATE', ['type' => $innerGLA])]; } else { $innerOutputs = $innerGLA->output(); grokit_assert(\count($innerOutputs) == \count($glaOutputAtts), 'Expected ' . \count($glaOutputAtts) . ' outputs fromm Inner GLA, got ' . \count($innerOutputs)); } $constState = lookupResource('GroupByState', ['gla' => $innerGLA, 'groups' => $gbyAtts, 'debug' => $debug]); // constructor argumetns are inherited from inner GLA $configurable = $innerGLA->configurable(); $reqStates = $innerGLA->req_states(); // We need to specially create the constructor string because apparently // declaring Type Name(); is a function declaration instead of a variable // declaration for some reason. $constructorParts = []; if ($configurable) { $constructorParts[] = 'jsonInit'; } if ($innerGLA->has_state()) { $constructorParts[] = 'innerState'; } $constructorString = \count($constructorParts) > 0 ? '(' . implode(', ', $constructorParts) . ')' : ''; // add the outputs we got from the gla foreach ($innerOutputs as $name => $type) { grokit_assert(array_key_exists($name, $outputs), 'Inner GLA\'s outputs refer to unknown attribute ' . $name); grokit_assert($type !== null, 'GroupBy Inner GLA left output ' . $name . ' with no type'); $outputs[$name] = $type; } $iterable = $innerGLA->iterable(); // need to keep track of system includes needed $extraHeaders = array(); $allocatorText = "std::allocator<std::pair<const Key, {$innerGLA}> >"; if ($use_mct) { $keepHashesText = $keepHashes ? 'true' : 'false'; $extraHeaders[] = "mct/hash-map.hpp"; $map = "mct::closed_hash_map<Key, {$innerGLA}, HashKey, std::equal_to<Key>, {$allocatorText}, {$keepHashesText}>"; $mapType = 'mct::closed_hash_map'; } else { $extraHeaders[] = "unordered_map"; $map = "std::unordered_map<Key, {$innerGLA}, HashKey, std::equal_to<Key>, {$allocatorText}>"; $mapType = 'std::unordered_map'; } if ($debug > 0) { $extraHeaders[] = 'cstdio'; } ?> class <?php echo $className; ?> { public: using ConstantState = <?php echo $constState; ?> ; <?php if ($innerGLA->has_state()) { ?> using InnerState = ConstantState::InnerState; <?php } // if gla has state ?> using Key = ConstantState::Key; using HashKey = ConstantState::HashKey; using InnerGLA = <?php echo $innerGLA; ?> ; typedef <?php echo $map; ?> MapType; static const size_t INIT_SIZE = <?php echo $init_size; ?> ; public: class Iterator { MapType::iterator it; // current value MapType::iterator end; // last value in the fragment public: Iterator() { } Iterator(MapType::iterator _it, MapType::iterator _end): it(_it), end(_end) { if( it != end ) { <?php switch ($innerRes) { case 'multi': ?> it->second.Finalize(); <?php break; case 'state': if ($innerGLA->finalize_as_state()) { ?> it->second.FinalizeState(); <?php } // if we need to finalize as a state break; } // end switch inner restype ?> } } bool GetNextResult( <?php echo typed_ref_args($outputs); ?> ) { bool gotResult = false; while( it != end && !gotResult ) { <?php echo $innerGLA; ?> & gla = it->second; <?php foreach ($gbyAttMap as $in => $out) { ?> <?php echo $out; ?> = it->first.<?php echo $in; ?> ; <?php } // foreach grouping attribute ?> <?php switch ($innerRes) { case 'multi': ?> gotResult = gla.GetNextResult( <?php echo args($innerOutputs); ?> ); if( !gotResult ) { ++it; if( it != end ) { it->second.Finalize(); } } <?php break; case 'single': ?> gotResult = true; gla.GetResult(<?php echo args($innerOutputs); ?> ); ++it; <?php break; case 'state': reset($innerOutputs); // Assuming that $innerOutputs contains a single value that is // the state type. $oName = key($innerOutputs); $oType = current($innerOutputs); ?> gotResult = true; <?php echo $oName; ?> = <?php echo $oType; ?> ( &gla ); ++it; <?php } // switch inner result type ?> } return gotResult; } }; private: const ConstantState & constState; <?php if ($configurable) { ?> const Json::Value jsonInit; <?php } // if configurable ?> size_t count; MapType groupByMap; std::vector<MapType::iterator> theIterators; // the iterators, only 2 elements if multi, many if fragment Iterator multiIterator; public: <?php echo $className; ?> (<?php if ($configurable) { ?> const Json::Value & _jsonInit, <?php } ?> const ConstantState & _constState ) : constState(_constState) <?php if ($configurable) { ?> , jsonInit(_jsonInit) <?php } // if configurable ?> , count(0) , groupByMap( INIT_SIZE ) , theIterators() , multiIterator() { } ~<?php echo $className; ?> () {} void Reset(void) { count = 0; groupByMap.clear(); theIterators.clear(); } void AddItem(<?php echo array_template('const {val} & {key}', ', ', $inputs); ?> ) { count++; // check if _key is already in the map; if yes, add _value; else, add a new // entry (_key, _value) Key key(<?php echo array_template('{key}', ', ', $gbyAtts); ?> ); MapType::iterator it = groupByMap.find(key); if (it == groupByMap.end()) { // group does not exist // create an empty GLA and insert // better to not add the item here so we do not have // to transport a large state <?php if ($innerGLA->has_state()) { ?> const InnerState & innerState = constState.getConstState(key); <?php } // if gla has state ?> InnerGLA gla<?php echo $constructorString; ?> ; auto ret = groupByMap.insert(MapType::value_type(key, gla)); it = ret.first; // reposition } it->second.AddItem(<?php echo array_template('{key}', ', ', $glaInputAtts); ?> ); } void AddState(<?php echo $className; ?> & other) { count += other.count; // scan other hash and insert or update content in this one for (MapType::iterator it = other.groupByMap.begin(); it != other.groupByMap.end(); ++it) { const Key& okey = it->first; <?php echo $innerGLA; ?> & ogla = it->second; MapType::iterator itt = groupByMap.find(okey); if (itt != groupByMap.end()) { // found the group <?php echo $innerGLA; ?> & gla = itt->second; gla.AddState(ogla); } else { // add the other group to this hash groupByMap.insert(MapType::value_type(okey, ogla)); } } } <?php if ($iterable) { ?> bool ShouldIterate(ConstantState& modibleState) { <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : ==== ShouldIterate ====\n"); <?php } // if debugging enabled ?> bool shouldIterate = false; for( MapType::iterator it = groupByMap.begin(); it != groupByMap.end(); ++it ) { const Key & key = it->first; InnerGLA & gla = it->second; <?php if ($innerGLA->has_state()) { ?> InnerState & innerState = modibleState.getModibleState(key); <?php } // if gla has state ?> bool glaRet = gla.ShouldIterate(innerState); shouldIterate = shouldIterate || glaRet; <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : Key(%s) shouldIterate(%s)\n", key.to_string().c_str(), glaRet ? "true" : "false"); <?php } // if debugging enabled ?> } return shouldIterate; } <?php } // if iterable ?> <?php if (in_array('fragment', $resType)) { ?> int GetNumFragments(void){ int size = groupByMap.size(); int sizeFrag = <?php echo $fragSize; ?> ; // setup the fragment boundaries // scan via iterator and count int frag=0; int pos=0; MapType::iterator it = groupByMap.begin(); theIterators.clear(); theIterators.push_back( it ); // special case when size < num_fragments // > if (sizeFrag == 0){ it = groupByMap.end(); theIterators.push_back( it ); return 1; // one fragment } while(it!=groupByMap.end()){ while(it!=groupByMap.end() && pos<( frag + 1 )*sizeFrag){ //> ++it; pos++; } theIterators.push_back( it ); frag++; } <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : fragments(%d)\n", frag); <?php } ?> return frag; } Iterator* Finalize(int fragment){ // Call finalize on all inner GLAs in this fragment. MapType::iterator iter = theIterators[fragment]; MapType::iterator iterEnd = theIterators[fragment+1]; Iterator* rez = new Iterator(theIterators[fragment], theIterators[fragment+1] ); return rez; } bool GetNextResult(Iterator* it, <?php echo array_template('{val} & {key}', ', ', $outputs); ?> ) { return it->GetNextResult(<?php echo args($outputs); ?> ); } <?php } // if using fragment interface ?> void Finalize() { multiIterator = Iterator( groupByMap.begin(), groupByMap.end() ); <?php if ($debug >= 1) { ?> fprintf(stderr, "<?php echo $className; ?> : groups(%lu) tuples(%lu)\n", groupByMap.size(), count); <?php } ?> } bool GetNextResult(<?php echo array_template('{val} & {key}', ', ', $outputs); ?> ) { return multiIterator.GetNextResult( <?php echo args($outputs); ?> ); } std::size_t size() const { return groupByMap.size(); } const MapType& GetMap() const { return groupByMap; } bool Contains(<?php echo const_typed_ref_args($gbyAtts); ?> ) const { Key key(<?php echo args($gbyAtts); ?> ); return groupByMap.count(key) > 0; } const InnerGLA& Get(<?php echo const_typed_ref_args($gbyAtts); ?> ) const { Key key(<?php echo args($gbyAtts); ?> ); return groupByMap.at(key); } bool Contains(Key key) const { return groupByMap.count(key) > 0; } const InnerGLA& Get(Key key) const { return groupByMap.at(key); } }; <?php if (in_array('fragment', $resType)) { ?> typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php } ?> <?php $sys_headers = array_merge(['iomanip', 'iostream', 'cstring'], $extraHeaders); return array('kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => array('HashFunctions.h'), 'input' => $inputs, 'output' => $outputs, 'result_type' => $resType, 'configurable' => $configurable, 'generated_state' => $constState, 'required_states' => $reqStates, 'iterable' => $iterable, 'properties' => ['resettable', 'finite container'], 'libraries' => $libraries, 'extra' => ['inner_gla' => $innerGLA]); }
function ConnectedComponents(array $t_args, array $inputs, array $outputs) { // Class name is randomly generated $className = generate_name("CCompGLA"); // Processing of inputs. grokit_assert(count($inputs) == 2, 'Connected Components: 2 inputs expected'); $inputs_ = array_combine(['src', 'dst'], $inputs); // Setting output type $outType = lookupType('int'); $outputs_ = ['node' => $outType, 'component' => $outType]; $outputs = array_combine(array_keys($outputs), $outputs_); $sys_headers = ["vector", "mct/hash-map.hpp"]; $user_headers = []; $lib_headers = []; ?> using namespace std; class <?php echo $className; ?> ; class <?php echo $className; ?> { class UnionFindMap{ private: mct::closed_hash_map<uint64_t, uint64_t>* parent; mct::closed_hash_map<uint64_t, uint64_t> sz; const uint64_t NON_EXISTING_ID = -1; public: // constructor did nothing UnionFindMap(){ parent = new mct::closed_hash_map<uint64_t, uint64_t>(); } uint64_t Find(uint64_t i){ if ((*parent).find(i) == (*parent).end()){ return NON_EXISTING_ID; } // use path compression here while (i != (*parent)[i]){ (*parent)[i] = (*parent)[(*parent)[i]]; i = (*parent)[i]; } return i; } // put merge small tree into higher tree // if disjoint, merge and return false void Union(uint64_t i, uint64_t j){ uint64_t ip = Find(i); uint64_t jp = Find(j); if (ip != NON_EXISTING_ID && jp != NON_EXISTING_ID){// both exists if (ip != jp){ if (sz[ip] < sz[jp]){ (*parent)[ip] = jp; sz[jp] += sz[ip]; }else{ (*parent)[jp] = ip; sz[ip] += sz[jp]; } } }else if(ip == NON_EXISTING_ID && jp == NON_EXISTING_ID){// both new (*parent)[i] = i; sz[i] = 2; (*parent)[j] = i; }else if (jp == NON_EXISTING_ID){ // i exists (*parent)[j] = ip; sz[ip] ++; }else{ (*parent)[i] = jp; sz[jp] ++; } } mct::closed_hash_map<uint64_t, uint64_t>* GetUF(){ return parent; } bool IsEmpty(){ return (*parent).empty(); } uint64_t GetSize(){ return (uint64_t) (*parent).size(); } void SetData(mct::closed_hash_map<uint64_t, uint64_t>* other_data){ parent = other_data; } // void FinalizeRoot(){ for(mct::closed_hash_map<uint64_t, uint64_t>::iterator it = (*parent).begin(); it != (*parent).end(); ++ it){ it->second = Find(it->first); } } void Clear(){ (*parent).clear(); sz.clear(); } ~UnionFindMap(){ delete parent; } }; private: // union-find map data structure, which contains nodeID->compID information UnionFindMap primary_uf; mct::closed_hash_map<uint64_t, uint64_t>::iterator output_iterator, output_iterator_end; bool localFinalized = false; public: <?php echo $className; ?> () {} void AddItem(<?php echo const_typed_ref_args($inputs_); ?> ) { uint64_t src_ = Hash(src); uint64_t dst_ = Hash(dst); primary_uf.Union(src_, dst_); } void AddState(<?php echo $className; ?> &other) { FinalizeLocalState(); other.FinalizeLocalState(); mct::closed_hash_map<uint64_t, uint64_t>* this_state_data = primary_uf.GetUF(); mct::closed_hash_map<uint64_t, uint64_t>* other_state_data = other.primary_uf.GetUF(); if (primary_uf.GetSize() < other.primary_uf.GetSize()){ mct::closed_hash_map<uint64_t, uint64_t>* tmp = this_state_data; this_state_data = other_state_data; other_state_data = tmp; primary_uf.SetData(this_state_data); other.primary_uf.SetData(other_state_data); } assert(primary_uf.GetSize() >= other.primary_uf.GetSize()); UnionFindMap secondary_uf; //go over the other state, and maintain a secondary table for(auto const& entry:(*other_state_data)){ if ((*this_state_data).count(entry.first) == 1){// key exists in this state uint64_t this_comp_id = (*this_state_data)[entry.first]; if (this_comp_id != entry.second) // merge needed secondary_uf.Union(this_comp_id, entry.second); }else{ (*this_state_data)[entry.first] = entry.second; } } // check if side table empty if (secondary_uf.IsEmpty()){ return; } // apply the side table secondary_uf.FinalizeRoot(); mct::closed_hash_map<uint64_t, uint64_t>* secondary_state_data = secondary_uf.GetUF(); for (auto& p:(*this_state_data)){ if ((*secondary_state_data).find(p.second) != (*secondary_state_data).end()){ p.second = (*secondary_state_data)[p.second]; } } } void FinalizeLocalState(){ if (!localFinalized){ primary_uf.FinalizeRoot(); localFinalized = true; } } void Finalize(){ output_iterator = primary_uf.GetUF()->begin(); output_iterator_end = primary_uf.GetUF()->end(); } bool GetNextResult(<?php echo typed_ref_args($outputs_); ?> ) { if (output_iterator != output_iterator_end){ node = output_iterator->first; component = output_iterator->second; ++ output_iterator; return true; }else{ return false; } } }; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'input' => $inputs, 'output' => $outputs, 'result_type' => 'multi']; }
function Max(array $t_args, array $input, array $output) { grokit_assert(\count($output) >= 1, 'Max GLA produces at least one output!'); grokit_assert(\count($output) == \count($input), 'Max GLA should have the same number of inputs and outputs'); $nValues = \count($output); $inputNames = array_keys($input); $outputNames = array_keys($output); // Outputs should be the same type as the inputs for ($index = 0; $index < $nValues; $index++) { array_set_index($output, $index, array_get_index($input, $index)); } $name = generate_name('Max_'); ?> class <?php echo $name; ?> { uintmax_t count; <?php foreach ($output as $k => $v) { ?> <?php echo $v; ?> _<?php echo $k; ?> ; <?php } // foreach output ?> public: <?php echo $name; ?> () : <?php foreach ($output as $k => $v) { ?> _<?php echo $k; ?> (), <?php } // foreach output ?> count(0) { } void AddItem( <?php echo const_typed_ref_args($input); ?> ) { if( count > 0 ) { <?php for ($index = 0; $index < $nValues; $index++) { ?> _<?php echo $outputNames[$index]; ?> = std::max(_<?php echo $outputNames[$index]; ?> , <?php echo $inputNames[$index]; ?> ); <?php } // foreach value ?> } else { <?php for ($index = 0; $index < $nValues; $index++) { ?> _<?php echo $outputNames[$index]; ?> = <?php echo $inputNames[$index]; ?> ; <?php } // foreach value ?> } count++; } void AddState( <?php echo $name; ?> & o ) { if (count > 0 && o.count > 0) { <?php for ($index = 0; $index < $nValues; $index++) { ?> _<?php echo $outputNames[$index]; ?> = std::max(_<?php echo $outputNames[$index]; ?> , o._<?php echo $outputNames[$index]; ?> ); <?php } // foreach value ?> } else if(o.count > 0) { // count == 0 <?php for ($index = 0; $index < $nValues; $index++) { ?> _<?php echo $outputNames[$index]; ?> = o._<?php echo $outputNames[$index]; ?> ; <?php } // foreach value ?> } // Otherwise, count > 0 && o.count == 0, so just keep our values count += o.count; } void GetResult(<?php echo typed_ref_args($output); ?> ) { <?php foreach ($output as $k => $v) { ?> <?php echo $k; ?> = _<?php echo $k; ?> ; <?php } // foreach output ?> } }; <?php return ['kind' => 'GLA', 'name' => $name, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'system_headers' => ['algorithm', 'cstdint']]; }
function ChunkCount(array $t_args, array $input, array $output) { $name = generate_name('ChunkCount'); grokit_assert(\count($output) == 2, 'ChunkCount only supports 2 outputs'); $numberName = array_keys($output)[0]; $countName = array_keys($output)[1]; $numberType = lookupType('BASE::BIGINT'); $countType = lookupType('BASE::BIGINT'); $output[$numberName] = $numberType; $output[$countName] = $countType; ?> class <?php echo $name; ?> { using map_type = std::unordered_map<<?php echo $numberType; ?> , <?php echo $countType; ?> >; map_type chunkCount; <?php echo $numberType; ?> curCount; map_type::const_iterator multiIter; public: <?php echo $name; ?> () : chunkCount(), curCount(0) { } void AddItem(<?php echo const_typed_ref_args($input); ?> ) { curCount++; } void AddState(const <?php echo $name; ?> & other) { for (auto elem: other.chunkCount) { chunkCount[elem.first] += elem.second; } } void ChunkBoundary() { chunkCount[curCount] += 1; curCount = 0; } void Finalize() { multiIter = chunkCount.cbegin(); } bool GetNextResult(<?php echo typed_ref_args($output); ?> ) { if (multiIter != chunkCount.cend()) { <?php echo $numberName; ?> = multiIter->first; <?php echo $countName; ?> = multiIter->second; multiIter++; return true; } else { return false; } } }; <?php return ['kind' => 'GLA', 'name' => $name, 'system_headers' => ['unordered_map'], 'input' => $input, 'output' => $output, 'result_type' => ['multi'], 'chunk_boundary' => true]; }
function CCompLP($t_args, $inputs, $outputs) { // Class name is randomly generated. $className = generate_name('CComp'); // Initializiation of argument names. $inputs_ = array_combine(['s', 't'], $inputs); $vertex = $inputs_['s']; // Construction of outputs. $outputs_ = ['node' => $vertex, 'component' => lookupType('int')]; $outputs = array_combine(array_keys($outputs), $outputs_); $sys_headers = ['armadillo', 'algorithm']; $user_headers = []; $lib_headers = []; $libraries = ['armadillo']; $properties = []; $extra = []; $result_type = ['multi']; ?> using namespace arma; using namespace std; class <?php echo $className; ?> ; <?php $constantState = lookupResource('graph::CCompLP_Constant_State', ['className' => $className]); ?> class <?php echo $className; ?> { public: // The constant state for this GLA. using ConstantState = <?php echo $constantState; ?> ; // The number of iterations to perform, not counting the initial set-up. static const constexpr int kIterations = 30; // The work is split into chunks of this size before being partitioned. static const constexpr int kBlock = 32; // The maximum number of fragments to use. static const constexpr int kMaxFragments = 64; private: // Node Component static arma::rowvec node_component; // The typical constant state for an iterable GLA. const ConstantState& constant_state; // The number of unique nodes seen. long num_nodes; // long output_iterator; // The current iteration. int iteration; // check if need more iterations long connections; public: <?php echo $className; ?> (const <?php echo $constantState; ?> & state) : constant_state(state), num_nodes(state.num_nodes), iteration(state.iteration),output_iterator(0),connections(0) { } // Basic dynamic array allocation. void AddItem(<?php echo const_typed_ref_args($inputs_); ?> ) { if (iteration == 0) { num_nodes = max((long) max(s, t), num_nodes); return; } /*else if (iteration == 1){ long max_known = (long) max(s, t); node_component(s) = max_known; node_component(t) = max_known; ++ connections; }*/else{ long s_id = node_component(s), t_id = node_component(t); if (s_id != t_id){ ++ connections; if (s_id > t_id) node_component(t) = s_id; else node_component(s) = t_id; } } } // Hashes are merged. void AddState(<?php echo $className; ?> &other) { if (iteration == 0) num_nodes = max(num_nodes, other.num_nodes); else connections += other.connections; } // Most computation that happens at the end of each iteration is parallelized // by performed it inside Finalize. bool ShouldIterate(ConstantState& state) { state.iteration = ++iteration; if (iteration == 1) {// allocate memory // num_nodes is incremented because IDs are 0-based. state.num_nodes = ++num_nodes; // Allocating space can't be parallelized. node_component.set_size(num_nodes); for (long i = 0; i < num_nodes; ++ i) node_component(i) = i; return true; } else { return connections > 0 && iteration < kIterations + 1; } } // Finalize does nothing void Finalize() {} bool GetNextResult(<?php echo typed_ref_args($outputs_); ?> ) { // should iterate is true if (connections > 0 && iteration < kIterations + 1) return false; if(output_iterator < num_nodes){ node = output_iterator++; component = node_component(node); return true; }else{ return false; } } }; // Initialize the static member types. arma::rowvec <?php echo $className; ?> ::node_component; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'properties' => $properties, 'extra' => $extra, 'iterable' => true, 'input' => $inputs, 'output' => $outputs, 'result_type' => $result_type, 'generated_state' => $constantState]; }
function Average(array $t_args, array $input, array $output) { $className = generate_name('Average'); grokit_assert(\count($input) == \count($output), 'Average must have the same number of inputs and outputs'); $outToIn = []; $internalTypes = []; $internalInit = []; reset($output); foreach ($input as $name => $type) { $outKey = key($output); $outToIn[$outKey] = $name; if ($type->is('numeric')) { $internalTypes[$name] = 'long double'; $internalInit[$name] = '0.0'; } else { $internalTypes[$name] = $type; $internalInit[$name] = ''; } if (is_null(current($output))) { if ($type->is('numeric')) { $output[$outKey] = lookupType('base::DOUBLE'); } else { $output[$outKey] = $type; } } next($output); } $countType = 'uint64_t'; $debug = get_default($t_args, 'debug', 0); ?> class <?php echo $className; ?> { private: <?php echo $countType; ?> count; // keeps the number of tuples aggregated <?php foreach ($internalTypes as $name => $type) { ?> <?php echo $type; ?> sum_<?php echo $name; ?> ; <?php } // foreach internal value ?> public: <?php echo $className; ?> () : count(0) <?php foreach ($internalInit as $name => $init) { ?> , sum_<?php echo $name; ?> (<?php echo $init; ?> ) <?php } // foreach internal initializer ?> {} void AddItem(<?php echo const_typed_ref_args($input); ?> ) { count++; <?php foreach ($input as $name => $type) { ?> sum_<?php echo $name; ?> += <?php echo $name; ?> ; <?php } // foreach input ?> } void AddState(<?php echo $className; ?> & o){ count += o.count; <?php foreach ($input as $name => $type) { ?> sum_<?php echo $name; ?> += o.sum_<?php echo $name; ?> ; <?php } // foreach input ?> } // we only support one tuple as output void GetResult(<?php echo typed_ref_args($output); ?> ) const { if( count > 0 ) { <?php foreach ($output as $name => $type) { $inName = $outToIn[$name]; ?> <?php echo $name; ?> = (sum_<?php echo $inName; ?> ) / count; <?php } // foreach output ?> } else { <?php foreach ($output as $name => $type) { ?> <?php echo $name; ?> = sum_<?php echo $inName; ?> ; <?php } // foreach output ?> } } }; <?php $sys_headers = ['cinttypes']; if ($debug > 0) { $sys_headers[] = 'iostream'; $sys_headers[] = 'sstream'; } return array('kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'input' => $input, 'output' => $output, 'result_type' => 'single'); }
function XMLReader(array $t_args, array $output) { $my_output = []; foreach ($output as $key => $out) { $name = $key; $my_output[$name] = $out; } $xpath_row = $t_args['paths']['row']; $xpath_columns = $t_args['paths']['columns']; $className = generate_name('XMLReader'); ?> class <?php echo $className; ?> { std::istream& my_stream; pugi::xml_document doc; const char* xpath_row; std::vector<const char*> xpath_columns; pugi::xpath_node_set rows; pugi::xpath_node_set::const_iterator start; pugi::xpath_node_set::const_iterator end; <?php \grokit\declareDictionaries($my_output); ?> public: <?php echo $className; ?> ( GIStreamProxy& _stream ) : my_stream(_stream.get_stream( )) { doc.load( my_stream ); xpath_row = "<?php echo $xpath_row; ?> "; rows = doc.select_nodes(xpath_row); <?php $push = 'xpath_columns.push_back("'; $close_brace = '");'; foreach ($xpath_columns as $col) { ?> <?php echo $push; echo $col; echo $close_brace; ?> <?php } ?> // Capture the first and last row start = rows.begin(); end = rows.end(); } bool ProduceTuple( <?php echo typed_ref_args($my_output); ?> ) { if( start == end ) { return false; } else { pugi::xml_node xmlnode = start->node(); std::vector<const char*>::const_iterator it = xpath_columns.begin(); <?php $col_num = 0; $node_type = 'auto '; $select_single_node = ' = xmlnode.select_single_node('; $close = ');'; $iterator_increment = '++it;'; foreach ($my_output as $name => $type) { ?> <?php echo $node_type; echo ' col_'; echo $col_num; echo $select_single_node; ?> *it<?php echo $close; ?> <?php echo \grokit\fromStringDict($name, $type, 'col_' . $col_num . '.node().child_value()'); ?> ; <?php $col_num = $col_num + 1; ?> <?php echo $iterator_increment; ?> <?php } ?> ++start; return true; } } }; <?php $sys_headers = ['vector', 'string', 'iostream', 'fstream']; return ['name' => $className, 'kind' => 'GI', 'output' => $my_output, 'system_headers' => $sys_headers, 'lib_headers' => ['pugixml.hpp']]; }
function Segmenter(array $t_args, array $input, array $output, array $given_states) { $resType = ['fragment', 'multi']; $system_headers = ['array', 'vector', 'memory', 'cinttypes', 'unordered_map']; $user_headers = ['HashFunctions.h']; $lib_headers = []; $preferFragment = get_default($t_args, 'inner.prefer.fragment', false); $wantedRes = $preferFragment ? ['fragment', 'multi'] : ['multi', 'fragment']; $nInputs = \count($input); grokit_assert($nInputs > 1, 'Segmenter: Not enough inputs specified!'); $keyName = array_keys($input)[0]; $keyType = array_get_index($input, 0); $innerInputs = array_slice($input, 1, $nInputs - 1, true); $gla = get_first_key($t_args, ['gla', 'GLA', 0]); grokit_assert(is_gla($gla), 'Segmenter: [gla] argument must be a valid GLA'); $gla = $gla->apply($innerInputs, $output, $given_states); $n_passes = get_default($t_args, 'passes', 1); grokit_assert(is_int($n_passes), 'Segmenter: [passes] argument must be an integer'); grokit_assert($n_passes > 0, 'Segmenter: [passes] argument must be > 0'); $libraries = $gla->libraries(); $innerRes = get_first_value($gla->result_type(), $wantedRes); $innerInputs = $gla->input(); $innerOutput = $gla->output(); $input = array_merge([$keyName => $keyType], $innerInputs); $output = $innerOutput; $segments = get_default($t_args, 'segments', 64); $constState = lookupResource('BASE::SegmenterState', ['gla' => $gla, 'passes' => $n_passes, 'segments' => $segments]); $className = generate_name('Segmenter_'); $savedArgs = []; $cArgs = []; $innerCArgs = []; if ($gla->configurable()) { $savedArgs['json_init'] = 'Json::Value'; $cArgs['json_init'] = 'Json::Value'; $innerCArgs[] = 'json_init'; } $cArgs['const_state'] = $constState; if ($gla->has_state()) { $innerCArgs[] = 'constState.inner_cstate'; } $cstStr = \count($innerCArgs) > 0 ? '(' . implode(',', $innerCArgs) . ')' : ''; grokit_assert(!$gla->iterable(), 'Segementer does not support iterable GLAs'); $iterable = $n_passes > 1; ?> class <?php echo $className; ?> { private: using ConstantState = <?php echo $constState; ?> ; using SplitState = ConstantState::SplitState; static constexpr const size_t NUM_STATES = SplitState::NUM_STATES; using InnerGLA = <?php echo $gla; ?> ; using InnerGLAPtr = std::unique_ptr<InnerGLA>; using GLA_Array = std::array<InnerGLAPtr, NUM_STATES>; public: using size_type = std::size_t; <?php if ($innerRes == 'fragment') { ?> class Iterator { private: InnerGLA * gla; int fragmentNum; InnerGLA::Iterator * innerIter; public: Iterator( InnerGLA * _gla, int _fragmentNum, int _innerFrag ) : gla(_gla), fragmentNum(_fragmentNum), innerIter(nullptr) { innerIter = gla->Finalize(_innerFrag); } ~Iterator(void) { if( innerIter != nullptr ) { delete innerIter; innerIter = nullptr; } } bool GetNextResult( <?php echo typed_ref_args($gla->output()); ?> ) { return innerIter->GetNextResult(<?php echo args($gla->output()); ?> ); } int FragmentNumber() { return fragmentNum; } }; <?php } else { // if inner result type is fragment ?> class Iterator { private: InnerGLA * gla; int fragmentNum; public: Iterator( InnerGLA * _gla, int fragNo ) : gla(_gla), fragmentNum(fragNo) { gla->Finalize(); } ~Iterator(void) { } bool GetNextResult( <?php echo typed_ref_args($gla->output()); ?> ) { return gla->GetNextResult(<?php echo args($gla->output()); ?> ); } int FragmentNumber() { return fragmentNum; } }; <?php } // if inner result type is multi ?> private: const ConstantState & constState; GLA_Array localState; // Iteration state for multi result type int numFrags; int multiFragNo; Iterator * multiIter; <?php if ($innerRes == 'fragment') { ?> using frag_info = std::pair<int, int>; using frag_map_t = std::unordered_map<int, frag_info>; frag_map_t fragMap; <?php } ?> <?php foreach ($savedArgs as $name => $type) { ?> const <?php echo $type; ?> <?php echo $name; ?> ; <?php } // foreach saved arg ?> public: // Constructor <?php echo $className; ?> ( <?php echo const_typed_ref_args($cArgs); ?> ) : constState(const_state) , localState() , numFrags(0) , multiFragNo(0) , multiIter(nullptr) <?php if ($innerRes == 'fragment') { ?> , fragMap() <?php } foreach ($savedArgs as $name => $type) { ?> , <?php echo $name; ?> (<?php echo $name; ?> ) <?php } // foreach constructor arg to save ?> { for( auto & elem : localState ) { elem.reset(new InnerGLA<?php echo $cstStr; ?> ); } } void AddItem( <?php echo const_typed_ref_args($input); ?> ) { uint64_t hashVal = CongruentHash(Hash(<?php echo $keyName; ?> ), H_b + 1); uint64_t passNum = (hashVal / NUM_STATES) % ConstantState::N_PASSES; uint64_t segNum = hashVal % NUM_STATES; <?php if ($n_passes > 1) { ?> if( passNum != constState.pass ) { return; } <?php } // more than 1 pass ?> localState[segNum]->AddItem(<?php echo args($innerInputs); ?> ); } void ChunkBoundary(void) { // Merge local states into the global state SplitState & globalStates = constState.segments; int theseAreOk[NUM_STATES]; for( int i = 0; NUM_STATES > i; i++ ) { theseAreOk[i] = 1; } int segsLeft = NUM_STATES; while( segsLeft > 0 ) { InnerGLA * checkedOut = nullptr; int whichOne = globalStates.CheckOutOne( theseAreOk, checkedOut ); if( checkedOut == NULL ) { checkedOut = new InnerGLA<?php echo $cstStr; ?> ; } checkedOut->AddState( *(localState[whichOne]) ); globalStates.CheckIn( whichOne, checkedOut ); theseAreOk[whichOne] = 0; segsLeft--; } // Re-initialize the local states for( auto & elem : localState ) { <?php if ($gla->is('resettable')) { ?> elem->Reset(); <?php } else { // if resettable ?> elem.reset(new InnerGLA<?php echo $cstStr; ?> ); <?php } // if not resettable ?> } } void AddState( <?php echo $className; ?> & o ) { // Do nothing } void Finalize() { SplitState & globalStates = constState.segments; if( multiIter != nullptr) delete multiIter; multiFragNo = 0; <?php if ($innerRes == 'fragment') { ?> frag_info fInfo = fragMap[multiFragNo]; multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo, fInfo.second); <?php } else { ?> multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo); <?php } ?> } bool GetNextResult(<?php echo typed_ref_args($output); ?> ) { bool gotResult = false; SplitState & globalStates = constState.segments; while( (multiFragNo < numFrags && multiIter != nullptr) && !gotResult ) { gotResult = multiIter->GetNextResult(<?php echo args($output); ?> ); if( !gotResult ) { multiFragNo++; delete multiIter; if( numFrags > multiFragNo ) { <?php if ($innerRes == 'fragment') { ?> frag_info fInfo = fragMap[multiFragNo]; multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo, fInfo.second); <?php } else { ?> multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo); <?php } ?> } else { multiIter = nullptr; } } } return gotResult; } int GetNumFragments(void) { <?php if ($innerRes == 'fragment') { ?> SplitState & globalStates = constState.segments; numFrags = 0; for (int i = 0; i < NUM_STATES; i++) { int curFrags = globalStates.Peek(i)->GetNumFragments(); for (int curFrag = 0; curFrag < curFrags; curFrag++) { fragMap[numFrags] = frag_info(i, curFrag); numFrags++; } } <?php } else { ?> numFrags = NUM_STATES; <?php } ?> return numFrags; } Iterator * Finalize( int fragment ) { SplitState & globalStates = constState.segments; <?php if ($innerRes == 'fragment') { ?> frag_info info = fragMap[fragment]; return new Iterator(globalStates.Peek(info.first), fragment, info.second); <?php } else { ?> return new Iterator(globalStates.Peek(fragment), fragment); <?php } ?> } bool GetNextResult( Iterator * it, <?php echo typed_ref_args($output); ?> ) { bool ret = it->GetNextResult(<?php echo args($output); ?> ); return ret; } <?php if ($iterable) { ?> bool ShouldIterate( ConstantState & modible ) { modible.pass++; return modible.pass < ConstantState::N_PASSES; } void PostFinalize() { constState.segments.Reset(); } <?php } // iterable ?> <?php if ($gla->is('finite container')) { ?> size_type size() { SplitState & globalStates = constState.segments; size_type s = 0; for( int i = 0; NUM_STATES > i; i++ ) { InnerGLA * ptr = globalStates.Peek(i); s += ptr->size(); } return s; } size_type size(int frag) { SplitState & globalStates = constState.segments; return globalStates.Peek(frag)->size(); } <?php } // if the gla is a container ?> }; typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $system_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'input' => $input, 'output' => $output, 'result_type' => $resType, 'generated_state' => $constState, 'required_states' => $gla->req_states(), 'chunk_boundary' => true, 'configurable' => $gla->configurable(), 'iterable' => $iterable, 'post_finalize' => $iterable, 'intermediates' => true]; }
function Multiplexer(array $t_args, array $inputs, array $outputs) { $className = generate_name('Multiplexer'); if (\count($inputs) == 0) { grokit_assert(array_key_exists('input', $t_args), 'No inputs specified for Multiplexer'); $inputs = $t_args['input']; foreach ($t_args['inputs'] as $name => &$type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Only types may be specified as inputs to Multiplexer.'); } $inputs = ensure_valid_names($inputs, 'multi_input'); } $glas = get_first_key($t_args, ['glas', 0]); grokit_assert(\count($glas) > 0, 'No GLAs specified for Multiplexer.'); $myGLAs = []; $glaInputs = []; $glaOutputs = []; $resultType = 'multi'; $usedOutputs = []; $libraries = []; $glaGenStates = []; $glaReqStates = []; $configurable = false; $constArgs = []; $genStates = []; $reqStates = []; $iterable = null; foreach ($glas as $name => $glaInfo) { grokit_assert(is_array($glaInfo), 'Template argument \'glas\' must be an array'); grokit_assert(array_key_exists('gla', $glaInfo), 'No GLA given for glas[' . $name . ']'); grokit_assert(array_key_exists('inputs', $glaInfo), 'No inputs given for glas[' . $name . ']'); grokit_assert(array_key_exists('outputs', $glaInfo), 'No outputs given for glas[' . $name . ']'); $gla = $glaInfo['gla']; $glaInAtts = $glaInfo['inputs']; $glaOutAtts = $glaInfo['outputs']; grokit_assert(is_gla($gla), 'Non-GLA given for glas[' . $name . '][gla]'); grokit_assert(is_array($glaInAtts), 'Non-array given for inputs for gla ' . $name); grokit_assert(is_array($glaOutAtts), 'Non-array given for outputs for gla ' . $name); $glaInAtts = array_map('strval', $glaInAtts); $glaOutAtts = array_map('strval', $glaOutAtts); $glaName = "innerGLA_" . $name; $glaInputs[$glaName] = []; $glaOutputs[$glaName] = []; foreach ($glaInAtts as $att) { grokit_assert(array_key_exists($att, $inputs), 'Input ' . $att . ' for GLA ' . $name . ' not found in inputs'); $glaInputs[$glaName][$att] = $inputs[$att]; } foreach ($glaOutAtts as $att) { grokit_assert(array_key_exists($att, $outputs), 'Output ' . $att . ' for GLA ' . $name . ' not found in outputs'); grokit_assert(!in_array($att, $usedOutputs), 'Output ' . $att . ' used by multiple GLAs'); $usedOutputs[] = $att; $glaOutputs[$glaName][$att] = $outputs[$att]; } //fwrite(STDERR, "Inputs for GLA " . $glaName . ": " . print_r($glaInputs[$glaName], true) . PHP_EOL ); //fwrite(STDERR, "Outputs for GLA " . $glaName . ": " . print_r($glaOutputs[$glaName], true) . PHP_EOL ); $gla = $gla->apply($glaInputs[$glaName], $glaOutputs[$glaName]); $myGLAs[$glaName] = $gla; $glaRez[$glaName] = get_first_value($gla->result_type(), ['multi', 'single', 'state']); $libraries = array_merge($libraries, $gla->libraries()); if ($glaRez[$glaName] == 'state') { grokit_assert(\count($glaOutputs[$glaName]) == 1, "GLA {$glaName} is produced as state, and thus must have exactly 1 output."); $stateType = lookupType('base::STATE', ['type' => $gla]); $glaOutputs[$glaName] = array_combine(array_keys($glaOutputs[$glaName]), [$stateType]); } else { grokit_assert(\count($glaOutputs[$glaName]) == \count($gla->output()), 'GLA ' . $glaName . ' produces different number of outputs than expected'); $glaOutputs[$glaName] = array_combine(array_keys($glaOutputs[$glaName]), $gla->output()); } // Set types for our output foreach ($glaOutputs[$glaName] as $attName => $type) { $outputs[$attName] = $type; } if (is_null($iterable)) { $iterable = $gla->iterable(); } else { grokit_assert($iterable == $gla->iterable(), 'Multiplexer does not support mixing iterable and non-iterable GLAs'); } $glaReqStates[$glaName] = $gla->req_states(); foreach ($gla->req_states() as $rstate) { $reqStates[] = $rstate; } $glaGenStates[$glaName] = $gla->state(); // TODO: Support constant states grokit_assert(!$gla->has_state(), 'Multiplexer currently does not support constant states.'); } $libraries = array_unique($libraries); $extra = ['glas' => $myGLAs]; ?> class <?php echo $className; ?> { <?php foreach ($myGLAs as $name => $type) { ?> <?php echo $type; ?> <?php echo $name; ?> ; <?php } // foreach inner gla ?> class Iterator { bool _gotResultsOnce; bool _valid; <?php foreach ($myGLAs as $name => $type) { ?> <?php echo $type; ?> * it_<?php echo $name; ?> ; <?php } // foreach inner gla ?> public: Iterator(void) : _gotResultsOnce(false), _valid(false), <?php echo array_template('it_{key}(nullptr)', ', ', $myGLAs); ?> { } Iterator(<?php echo typed_ref_args($myGLAs); ?> ) : _gotResultsOnce(false), _valid(true), <?php echo array_template('it_{key}(&{key})', ', ', $myGLAs); ?> { <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'multi') { ?> <?php echo $name; ?> .Finalize(); <?php } // if inner GLA is multi } // foreach inner gla ?> } Iterator( const Iterator & other) = default; ~Iterator() { } bool GetNextResult( <?php echo typed_ref_args($outputs); ?> ) { FATALIF(!_valid, "Tried to get results from an invalid iterator."); bool ret = !_gotResultsOnce; _gotResultsOnce = true; <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'multi') { ?> ret |= it_<?php echo $name; ?> ->GetNextResult(<?php echo args($glaOutputs[$name]); ?> ); <?php } // if inner GLA is multi } // foreach inner gla ?> if( ret ) { <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'single') { ?> it_<?php echo $name; ?> ->GetResult(<?php echo args($glaOutputs[$name]); ?> ); <?php } else { if ($glaRez[$name] == 'state') { $stateVar = array_keys($glaOutputs[$name])[0]; $stateType = $glaOutputs[$name][$stateVar]; ?> <?php echo $stateVar; ?> = <?php echo $stateType; ?> (it_<?php echo $name; ?> ); <?php } } // if inner GLA is state } // foreach inner gla ?> } return ret; } }; Iterator multiIterator; public: <?php echo $className; ?> () { } ~<?php echo $className; ?> () { } void AddItem(<?php echo const_typed_ref_args($inputs); ?> ) { // Call AddItem individually on each GLA. <?php foreach ($myGLAs as $gName => $gType) { ?> <?php echo $gName; ?> .AddItem(<?php echo args($glaInputs[$gName]); ?> ); <?php } // foreach inner gla ?> } void AddState( <?php echo $className; ?> & other ) { // Call AddState individually on each GLA. <?php foreach ($myGLAs as $gName => $gType) { ?> <?php echo $gName; ?> .AddState(other.<?php echo $gName; ?> ); <?php } // foreach inner gla ?> } void Finalize() { multiIterator = Iterator(<?php echo args($myGLAs); ?> ); } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { return multiIterator.GetNextResult(<?php echo args($outputs); ?> ); } void GetResult(<?php echo typed_ref_args($outputs); ?> ) { Finalize(); GetNextResult(<?php echo args($outputs); ?> ); } <?php foreach (array_keys($myGLAs) as $index => $name) { ?> const <?php echo $myGLAs[$name]; ?> & GetGLA<?php echo $index; ?> () const { return <?php echo $name; ?> ; } <?php } ?> }; <?php return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => $resultType, 'libraries' => $libraries, 'configurable' => $configurable, 'extra' => $extra); }
function page_rank_vd($t_args, $inputs, $outputs) { $className = generate_name('PageRank'); $inputs_ = array_combine(['s', 't'], $inputs); $vertex = $inputs_['s']; $outputs_ = ['node' => $vertex, 'rank' => lookupType('double')]; $outputs = array_combine(array_keys($outputs), $outputs_); $sys_headers = ['vector', 'algorithm']; $user_headers = []; $lib_headers = []; $libraries = []; $properties = []; $extra = []; $result_type = ['fragment']; ?> using namespace std; class <?php echo $className; ?> ; <?php $constantState = lookupResource('pagerankVD::page_rank_vd_Constant_State', ['className' => $className]); ?> class <?php echo $className; ?> { public: using ConstantState = <?php echo $constantState; ?> ; using Iterator = std::pair<int, int>; static const constexpr float kDamping = 0.85; static const constexpr int nIterations = 5; static const constexpr int kBlock = 32; static const constexpr int kMaxFragments = 64; private: static std::vector<double> weight; static std::vector<double> pagerank; static std::vector<double> oldPagerank; const ConstantState& constant_state; long nVertices; int iteration; int num_fragments; public: <?php echo $className; ?> (const <?php echo $constantState; ?> & state) : constant_state(state), nVertices(state.nVertices), iteration(state.iteration) { } void AddItem(<?php echo const_typed_ref_args($inputs_); ?> ) { if (iteration == 0) { nVertices = max((long) max(s, t), nVertices); return; } else if (iteration == 1) { weight[s]++; } else { oldPagerank[t] += weight[s] * pagerank[s]; } } void AddState(<?php echo $className; ?> &other) { if (iteration == 0) nVertices = max(nVertices, other.nVertices); } // Most computation that happens at the end of each iteration is parallelized // by performed it inside Finalize. bool ShouldIterate(ConstantState& state) { state.iteration = ++iteration; cout << "Finished Iteration: " << iteration << endl; if (iteration == 1) { state.nVertices = ++nVertices; cout << "num_nodes: " << nVertices << endl; oldPagerank.reserve(nVertices); pagerank.reserve(nVertices); weight.reserve(nVertices); std::fill (pagerank.begin(),pagerank.end(),1); return true; } else { return iteration < nIterations + 1; } } int GetNumFragments() { long size = (nVertices - 1) / kBlock + 1; // nVertices / kBlock rounded up. num_fragments = (iteration == 0) ? 0 : min(size, (long) kMaxFragments); cout << "Returning " << num_fragments << " fragments" << endl; return num_fragments; } Iterator* Finalize(long fragment) { int count = nVertices; int first = fragment * (count / kBlock) / num_fragments * kBlock; int final = (fragment == num_fragments - 1) ? count - 1 : (fragment + 1) * (count / kBlock) / num_fragments * kBlock - 1; if (iteration == 2) { for(int i = first; i <= final; i++){ pagerank[i] = 1; weight[i] = 1/weight[i]; oldPagerank[i] = 0; } } else { for(int i = first; i <= final; i++){ pagerank[i] = (1 - kDamping) + kDamping * oldPagerank[i]; oldPagerank[i] = 0; } } return new Iterator(first, final); } bool GetNextResult(Iterator* it, <?php echo typed_ref_args($outputs_); ?> ) { if (iteration < nIterations + 1) return false; if (it->first > it->second) return false; node = it->first; rank = pagerank[it->first]; it->first++; return true; } }; // Initialize the static member types. vector<double> <?php echo $className; ?> ::weight; vector<double> <?php echo $className; ?> ::pagerank; vector<double> <?php echo $className; ?> ::oldPagerank; typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'properties' => $properties, 'extra' => $extra, 'iterable' => true, 'intermediates' => true, 'input' => $inputs, 'output' => $outputs, 'result_type' => $result_type, 'generated_state' => $constantState]; }
function Gather(array $t_args, array $inputs, array $outputs) { // Class name randomly generated $className = generate_name("Gather"); $outputs = array_combine(array_keys($outputs), $inputs); $sys_headers = ['vector']; $user_headers = []; $lib_headers = []; $libraries = []; $properties = ['list']; $extra = []; $result_type = 'single'; ?> class <?php echo $className; ?> ; class <?php echo $className; ?> { public: using Tuple = std::tuple<<?php echo typed($inputs); ?> >; using Vector = std::vector<Tuple>; private: // The container that gathers the input. Vector items; // The tuple to be filled with the current item. Tuple item; // Used for iterating over the container during GetNextResult. int return_counter; public: <?php echo $className; ?> (){} void AddItem(<?php echo const_typed_ref_args($inputs); ?> ) { <?php foreach (array_keys($inputs) as $index => $name) { ?> std::get<<?php echo $index; ?> >(item) = <?php echo $name; ?> ; <?php } ?> items.push_back(item); } void AddState(<?php echo $className; ?> & other) { items.reserve(items.size() + other.items.size()); items.insert(items.end(), other.items.begin(), other.items.end()); } void Finalize() { return_counter = 0; } void GetResult(<?php echo typed_ref_args($outputs); ?> , int index = 0) const { <?php foreach (array_keys($outputs) as $index => $name) { ?> <?php echo $name; ?> = std::get<<?php echo $index; ?> >(items[index]); <?php } ?> } int GetCount() const { return items.size(); } const Vector& GetList() const { return items; } }; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'properties' => $properties, 'extra' => $extra, 'iterable' => false, 'input' => $inputs, 'output' => $outputs, 'result_type' => $result_type]; }
function OrderBy(array $t_args, array $inputs, array $outputs) { if (\count($inputs) == 0) { grokit_assert(array_key_exists('input', $t_args), 'No inputs given for OrderBy'); $inputs = $t_args['input']; foreach ($t_args['input'] as $name => &$type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Invalid type given for input ' . $name); } } grokit_assert(array_key_exists('order', $t_args), 'No ordering attributes given for OrderBy'); $ordering = $t_args['order']; $ascOpts = ['ASC', 'ASCENDING', '+', '>']; $descOpts = ['DESC', 'DESCENDING', 'DES', 'DSC', '-', '<']; $ascending = []; foreach ($ordering as $name => $order) { grokit_assert(array_key_exists($name, $inputs), 'Ordering attribute ' . $name . ' not present in input'); if (in_array_icase($order, $ascOpts)) { $ascending[$name] = true; } else { if (in_array_icase($order, $descOpts)) { $ascending[$name] = false; } else { grokit_error("Unknown ordering " . $order . " given for attribute " . $name); } } } $rankAtt = get_default($t_args, 'rank', null); grokit_assert(is_null($rankAtt) || is_attribute($rankAtt), 'Rank argument should be null or an attribute'); grokit_assert(is_null($rankAtt) || array_key_exists($rankAtt->name(), $outputs), 'Rank attribute does not exist in outputs'); if (!is_null($rankAtt) && is_null($outputs[$rankAtt->name()])) { $outputs[$rankAtt->name()] = lookupType('base::BIGINT'); } $outputPassthroughAtts = []; foreach ($outputs as $name => $type) { if (is_null($rankAtt) || $rankAtt->name() != $name) { $outputPassthroughAtts[$name] = $type; } } $outToIn = []; $nInputs = \count($inputs); reset($inputs); reset($outputPassthroughAtts); for ($i = 0; $i < $nInputs; $i++) { $outName = key($outputPassthroughAtts); $inName = key($inputs); $outToIn[$outName] = $inName; // Unify types $outputs[$outName] = $inputs[$inName]; $outputPassthroughAtts[$outName] = $inputs[$inName]; next($inputs); next($outputPassthroughAtts); } $orderAtts = []; $extraAtts = []; foreach ($inputs as $name => $type) { if (array_key_exists($name, $ordering)) { $orderAtts[$name] = $type; } else { $extraAtts[$name] = $type; } } // Give 2^32 as the default, which should be effectively infinite $limitDefault = pow(2, 32); $limit = get_default($t_args, 'limit', $limitDefault); $limit = $limit == 0 ? $limitDefault : $limit; grokit_assert($limit > 0, 'The OrderBy limit must be a positive integer'); $className = generate_name('OrderBy'); $debug = get_default($t_args, 'debug', 0); ?> class <?php echo $className; ?> { struct Tuple { <?php foreach ($inputs as $name => $type) { ?> <?php echo $type; ?> <?php echo $name; ?> ; <?php } ?> Tuple( void ) = default; Tuple( const Tuple & other ) = default; Tuple( <?php echo array_template('const {val} & _{key}', ', ', $inputs); ?> ): <?php echo array_template('{key}(_{key})', ', ', $inputs); ?> { } Tuple & operator = (const Tuple & other ) = default; bool operator > ( const Tuple & other ) const { <?php foreach ($orderAtts as $name => $type) { $op1 = $ascending[$name] ? '<' : '>'; $op2 = !$ascending[$name] ? '<' : '>'; ?> if( <?php echo $name; ?> <?php echo $op1; ?> other.<?php echo $name; ?> ) return true; else if( <?php echo $name; ?> <?php echo $op2; ?> other.<?php echo $name; ?> ) return false; <?php } ?> return false; } bool operator < ( const Tuple& other ) const { return other > *this; } bool operator <= (const Tuple & other ) const { return ! (*this > other ); } bool operator >= (const Tuple & other ) const { return !( other > *this ); } <?php if ($debug > 0) { ?> std::string toString(void) const { std::ostringstream ss; ss << "( "; // > <?php $first = true; foreach ($inputs as $name => $type) { if ($first) { $first = false; } else { echo ' ss << ", ";' . PHP_EOL; } ?> ss << <?php echo $name; ?> ; // > <?php } // foreach input ?> ss << " )"; // > return ss.str(); } <?php } // debug > 0 ?> }; // struct Tuple typedef std::vector<Tuple> TupleVector; public: class Iterator { public: typedef TupleVector::const_iterator iter_type; private: iter_type begin; iter_type curr; iter_type end; public: Iterator(void) = default; Iterator( const iter_type & _begin, const iter_type & _end ) : begin(_begin), curr(_begin), end(_end) { } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { if( curr != end ) { <?php foreach ($outputPassthroughAtts as $name => $type) { ?> <?php echo $name; ?> = curr-><?php echo $outToIn[$name]; ?> ; <?php } if (!is_null($rankAtt)) { ?> <?php echo $rankAtt; ?> = (curr - begin) + 1; <?php } // if we need to output the rank ?> curr++; return true; } else { return false; } } }; private: uintmax_t __count; // number of tuples covered // K, as in Top-K static constexpr size_t K = <?php echo $limit; ?> ; TupleVector tuples; // Iterator for multi output type Iterator multiIterator; typedef std::greater<Tuple> TupleCompare; // Function to force sorting so that GetNext gets the tuples in order. void Sort(void) { TupleCompare comp; // If tuples doesn't contain at least K elements, it was never made into // a heap in the first place, so sort it normally. if( tuples.size() >= K ) { std::sort_heap(tuples.begin(), tuples.end(), comp); } else { std::sort(tuples.begin(), tuples.end(), comp); } } // Internal function to add a tuple to the heap void AddTupleInternal(Tuple & t ) { <?php if ($debug >= 1) { ?> { std::ostringstream ss; ss << "T ACK: " << t.toString() << std::endl; // > std::cerr << ss.str(); // > } <?php } ?> TupleCompare comp; if( tuples.size() >= K ) { <?php if ($debug >= 1) { ?> { std::ostringstream ss; ss << "T REP: " << tuples.front().toString() << std::endl; // > std::cerr << ss.str(); // > } <?php } ?> std::pop_heap(tuples.begin(), tuples.end(), comp); tuples.pop_back(); tuples.push_back(t); std::push_heap(tuples.begin(), tuples.end(), comp); } else { tuples.push_back(t); if( tuples.size() == K ) { std::make_heap(tuples.begin(), tuples.end(), comp); } } } public: <?php echo $className; ?> () : __count(0), tuples(), multiIterator() { } ~<?php echo $className; ?> () { } void AddItem(<?php echo const_typed_ref_args($inputs); ?> ) { __count++; Tuple t(<?php echo args($inputs); ?> ); <?php if ($debug >= 2) { ?> { std::ostringstream ss; ss << "T NEW: " << t.toString() << std::endl; // > std::cerr << ss.str(); // > } <?php } ?> if( tuples.size() == K && !(t > tuples.front()) ) return; AddTupleInternal(t); } void AddState( <?php echo $className; ?> & other ) { __count += other.__count; for( Tuple & el : other.tuples ) { if( tuples.size() < K /*>*/ || el > tuples.front() ) { AddTupleInternal(el); } } } void Finalize() { Sort(); Iterator::iter_type begin = tuples.cbegin(); Iterator::iter_type end = tuples.cend(); multiIterator = Iterator(begin, end); <?php if ($debug >= 1) { ?> std::ostringstream ss; ss << "[ "; //> bool first = true; for( auto el : tuples ) { if( first ) first = false; else ss << ", "; //>> ss << el.toString(); //>> } ss << " ]" << std::endl; // > std::cerr << ss.str(); //>> <?php } ?> } bool GetNextResult( <?php echo typed_ref_args($outputs); ?> ) { return multiIterator.GetNextResult(<?php echo args($outputs); ?> ); } }; <?php $system_headers = ['vector', 'algorithm', 'cinttypes']; if ($debug > 0) { $system_headers = array_merge($system_headers, ['iostream', 'sstream', 'string']); } return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => 'multi', 'system_headers' => $system_headers); }
function Join($t_args, $inputs, $outputs, $states) { // Class name randomly generated $className = generate_name('Join'); // Processing of states; $states_ = array_combine(['mapping'], $states); // Processing of outputs. $values = $states_['mapping']->get('vals'); $outputs = array_combine(array_keys($outputs), $values); $sys_headers = ['map', 'tuple']; $user_headers = []; $lib_headers = []; $libraries = []; $properties = ['list']; $extra = []; $result_type = ['multi']; ?> class <?php echo $className; ?> ; <?php $constantState = lookupResource("Join_Constant_State", ['className' => $className, 'states' => $states]); ?> class <?php echo $className; ?> { private: using ConstantState = <?php echo $constantState; ?> ; using Mapping = ConstantState::Mapping; using Iter = Mapping::Map::const_iterator; // The GroupBy containing the hash. const <?php echo $constantState; ?> & constant_state; // The iterator used for the multi return. Iter it, end; public: <?php echo $className; ?> (const <?php echo $constantState; ?> & state) : constant_state(state) { } void ProcessTuple(<?php echo const_typed_ref_args($inputs); ?> ) { auto key = Mapping::ChainHash(<?php echo args($inputs); ?> ); auto pair = constant_state.map.equal_range(key); it = pair.first; end = pair.second; } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { if (it == end) return false; <?php foreach (array_keys($outputs) as $index => $name) { ?> <?php echo $name; ?> = std::get<<?php echo $index; ?> >(it->second); <?php } ?> ++it; return true; } }; <?php return ['kind' => 'GT', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'input' => $inputs, 'output' => $outputs, 'result_type' => 'multi', 'generated_state' => $constantState]; }
/** * A GLA that determines the distinct values of a dataset. */ function Distinct(array $t_args, array $input, array $output) { grokit_assert(\count($input) == \count($output), 'Distinct must have the same outputs as inputs.'); $outputsToInputs = []; $i = 0; foreach ($input as $name => $type) { $outputsToInputs[array_keys($output)[$i]] = $name; array_set_index($output, $i++, $type); } $useMCT = get_default($t_args, 'use.mct', true); $initSize = get_default($t_args, 'init.size', 65536); $keepHashes = get_default($t_args, 'mct.keep.hashes', false); $fragmentSize = get_default($t_args, 'fragment.size', 100000); $nullCheck = get_default($t_args, 'null.check', false); grokit_assert(is_bool($useMCT), 'Distinct use.mct argument must be boolean'); grokit_assert(is_integer($initSize), 'Distinct init.size argument must be an integer'); grokit_assert($initSize > 0, 'Distinct init.size argument must be positive'); grokit_assert(is_bool($keepHashes), 'Distinct mct.keep.hashes argument must be boolean'); grokit_assert(is_integer($fragmentSize), 'Distinct fragment.size argument must be integral'); grokit_assert($fragmentSize > 0, 'Distinct fragment.size argumenst must be positive'); $nullable = []; if (is_bool($nullCheck)) { foreach ($input as $name => $type) { $nullable[$name] = $nullCheck; } } else { if (is_array($nullCheck)) { foreach ($input as $name => $type) { $nullable[$name] = false; } foreach ($nullCheck as $index => $n) { grokit_assert(is_string($n), 'Distinct null.check has invalid value at position ' . $index); grokit_assert(array_key_exists($n, $nullable), 'Distinct null.check has unknown input ' . $n . ' at position ' . $index); $nullable[$n] = true; } } else { grokit_error('Distinct null.check must be boolean or list of inputs to check for nulls'); } } $keepHashesText = $keepHashes ? 'true' : 'false'; $system_headers = ['cinttypes', 'functional', 'vector']; if ($useMCT) { $system_headers[] = 'mct/hash-set.hpp'; $definedSet = "mct::closed_hash_set<Key, HashKey, std::equal_to<Key>, std::allocator<Key>, {$keepHashesText}>"; } else { $system_headers[] = 'unordered_map'; $definedSet = "std::unordered_set<Key, HashKey, std::equal_to<Key>, std::allocator<Key>>"; } $className = generate_name('Distinct'); ?> class <?php echo $className; ?> { public: // Value being placed into the set. struct Key { <?php foreach ($input as $name => $type) { ?> <?php echo $type; ?> <?php echo $name; ?> ; <?php } // for each input ?> // Construct the value by copying all of the attributes. Key(<?php echo const_typed_ref_args($input); ?> ) : <?php $first = true; foreach ($input as $name => $type) { ?> <?php echo $first ? ' ' : ','; ?> <?php echo $name; ?> (<?php echo $name; ?> ) <?php $first = false; } // for each input ?> { } bool operator==(const Key & o ) const { return true <?php echo array_template("&& ({key} == o.{key})", ' ', $input); ?> ; } size_t hash_value() const { uint64_t hash = H_b; <?php foreach ($input as $name => $type) { ?> hash = CongruentHash(Hash(<?php echo $name; ?> ), hash); <?php } // for each input ?> return size_t(hash); } }; // Hashing functor for our value struct HashKey { size_t operator()(const Key& o) const { return o.hash_value(); } }; using Set = <?php echo $definedSet; ?> ; // Iterator object used in multi and fragment result types class Iterator { public: using iterator_t = Set::const_iterator; private: iterator_t start; iterator_t end; public: Iterator() : start(), end() { } Iterator( const iterator_t & _start, const iterator_t & _end ) : start(_start), end(_end) { } Iterator( const Iterator & o ) : start(o.start), end(o.end) { } bool GetNextResult(<?php echo typed_ref_args($output); ?> ) { if( start != end ) { <?php foreach ($output as $name => $type) { ?> <?php echo $name; ?> = start-><?php echo $outputsToInputs[$name]; ?> ; <?php } // for each output ?> start++; return true; } else { return false; } } }; private: // Constants static constexpr size_t INIT_SIZE = <?php echo $initSize; ?> ; static constexpr size_t FRAG_SIZE = <?php echo $fragmentSize; ?> ; // Member variables uint64_t count; // Total # tuples seen Set distinct; // Set of distinct values using IteratorList = std::vector<Iterator>; Iterator multiIterator; // Internal iterator for multi result type IteratorList fragments; // Iterator for fragments public: <?php echo $className; ?> () : count(0), distinct(INIT_SIZE), multiIterator(), fragments() { } ~<?php echo $className; ?> () { } void Reset(void) { count = 0; distinct.clear(); } void AddItem(<?php echo const_typed_ref_args($input); ?> ) { count++; <?php foreach ($nullable as $name => $check) { if ($check) { ?> if( IsNull( <?php echo $name; ?> ) ) return; <?php } // if checking for nulls } // foreach input ?> Key key(<?php echo args($input); ?> ); distinct.insert(key); /* auto it = distinct.find(key); if( it == distinct.end() ) { distinct.insert(key); } */ } void AddState( <?php echo $className; ?> & other ) { for( auto & elem : other.distinct ) { distinct.insert(elem); /* auto it = distinct.find(elem); if( it == distinct.end() ) { distinct.insert(elem); } */ } count += other.count; } // Multi interface void Finalize(void) { multiIterator = Iterator(distinct.cbegin(), distinct.cend()); } bool GetNextResult(<?php echo typed_ref_args($output); ?> ) { return multiIterator.GetNextResult(<?php echo args($output); ?> ); } // Fragment interface int GetNumFragments(void) { fragments.clear(); int nFrag = 0; Iterator::iterator_t prev = distinct.cbegin(); Iterator::iterator_t end = distinct.cend(); Iterator::iterator_t next = prev; while( next != end ) { for( size_t i = 0; next != end && FRAG_SIZE > i; i++ ) { next++; } Iterator nIter(prev, next); fragments.push_back(nIter); prev = next; nFrag++; } return nFrag; } Iterator * Finalize(int fragment) { return new Iterator(fragments[fragment]); } bool GetNextResult(Iterator * it, <?php echo typed_ref_args($output); ?> ) { return it->GetNextResult(<?php echo args($output); ?> ); } // General methods uint64_t get_count() const { return count; } uint64_t get_countDistinct() const { return distinct.size(); } const Set & get_distinct() const { return distinct; } }; typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => ['multi', 'fragment'], 'user_headers' => ['HashFunctions.h'], 'system_headers' => $system_headers, 'properties' => ['resettable']]; }
function CCompUnionFindLPFrag($t_args, $inputs, $outputs) { // Class name is randomly generated. $className = generate_name('CCompUnionFindLPFrag'); // Initializiation of argument names. $inputs_ = array_combine(['s', 't'], $inputs); $vertex = $inputs_['s']; // Construction of outputs. $outputs_ = ['node' => $vertex, 'component' => lookupType('base::BIGINT')]; $outputs = array_combine(array_keys($outputs), $outputs_); $sys_headers = ['armadillo', 'algorithm']; $user_headers = []; $lib_headers = []; $libraries = ['armadillo']; $properties = []; $extra = []; $result_type = ['fragment']; ?> using namespace arma; using namespace std; class <?php echo $className; ?> ; <?php $constantState = lookupResource('graph::CCompUnionFindLPFrag_Constant_State', ['className' => $className]); ?> class <?php echo $className; ?> { public: // The constant state for this GLA. using ConstantState = <?php echo $constantState; ?> ; // The current and final indices of the result for the given fragment. using Iterator = std::pair<uint64_t, uint64_t>; // The number of iterations to perform, not counting the initial set-up. static const constexpr int kIterations = 10; // The work is split into chunks of this size before being partitioned. static const constexpr int kBlock = 32; // The maximum number of fragments to use. static const constexpr int kMaxFragments = 64; private: // Node Component static arma::rowvec node_component; // keep track of component size static arma::rowvec component_size; // The typical constant state for an iterable GLA. const ConstantState& constant_state; // The number of unique nodes seen. uint64_t num_nodes; // The current iteration. int iteration; // The number of fragmetns for the result. int num_fragments; // check if need more iterations uint64_t connections; public: <?php echo $className; ?> (const <?php echo $constantState; ?> & state) : constant_state(state), num_nodes(state.num_nodes), iteration(state.iteration),connections(0) { } uint64_t Find(uint64_t node_id){ // use path compression here while (node_id != node_component(node_id)){ node_component(node_id) = node_component(node_component(node_id)); node_id = node_component(node_id); } return node_id; } void Union(uint64_t pid, uint64_t qid){ // find their root pid = Find(pid); qid = Find(qid); if (pid == qid) return; ++ connections; uint64_t psz = component_size(pid), qsz = component_size(qid); if (psz > qsz){ node_component(qid) = pid; component_size(pid) += qsz; }else{ node_component(pid) = qid; component_size(qid) += psz; } } // Basic dynamic array allocation. void AddItem(<?php echo const_typed_ref_args($inputs_); ?> ) { if (iteration == 0) { num_nodes = max((uint64_t) max(s, t), num_nodes); return; } else{ Union((uint64_t) s, (uint64_t) t); } } // Hashes are merged. void AddState(<?php echo $className; ?> &other) { if (iteration == 0) num_nodes = max(num_nodes, other.num_nodes); else connections += other.connections; } // Most computation that happens at the end of each iteration is parallelized // by performed it inside Finalize. bool ShouldIterate(ConstantState& state) { state.iteration = ++iteration; printf("Entering ShouldIterate. connections: %lu, iteration: %d\n", connections, iteration); if (iteration == 1) {// allocate memory // num_nodes is incremented because IDs are 0-based. state.num_nodes = ++num_nodes; // Allocating space can't be parallelized. node_component.set_size(num_nodes); for (uint64_t i = 0; i < num_nodes; ++ i){ node_component(i) = i; } component_size.set_size(num_nodes); component_size.fill(1); return true; } else { return connections > 0 && iteration < kIterations + 1; } } int GetNumFragments() { uint64_t size = (num_nodes - 1) / kBlock + 1; // num_nodes / kBlock rounded up. num_fragments = (iteration == 0) ? 0 : min(size, (uint64_t) kMaxFragments); printf("num_nodes: %lu, size: %lu, num_fragments: %d\n", num_nodes, size, num_fragments); return num_fragments; } // Finalize does nothing Iterator* Finalize(int fragment) { uint64_t count = num_nodes; // The ordering of operations is important. Don't change it. uint64_t first = fragment * (count / kBlock) / num_fragments * kBlock; uint64_t final = (fragment == num_fragments - 1) ? count - 1 : (fragment + 1) * (count / kBlock) / num_fragments * kBlock - 1; printf("fragment: %lu\tcount: %lu\tfirst: %lu\tfinal: %lu\n", fragment, count, first, final); return new Iterator(first, final); } bool GetNextResult(Iterator* it, <?php echo typed_ref_args($outputs_); ?> ) { // should iterate is true if (connections > 0 && iteration < kIterations + 1){ printf("I need more iterations, because connections: %lu, iteration: %d\n", connections, iteration); return false; } if (it->first > it->second) return false; node = it->first++; component = Find(node); return true; } }; // Initialize the static member types. arma::rowvec <?php echo $className; ?> ::node_component; arma::rowvec <?php echo $className; ?> ::component_size; typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'properties' => $properties, 'extra' => $extra, 'iterable' => true, 'input' => $inputs, 'output' => $outputs, 'result_type' => $result_type, 'generated_state' => $constantState]; }
function CSVReader(array $t_args, array $output) { $my_output = []; // Handle case where outputs are given as template arguments // and not implied. if (\count($output) == 0) { grokit_assert(array_key_exists('output', $t_args), 'Did not receive any description of my output!'); $output_list = $t_args['output']; grokit_assert(is_array($output_list), 'Expected list of types for template argument "output"'); $i = 1; foreach ($outputs_list as $name => $out_type) { grokit_assert(is_datatype($out_type) || is_identifier($out_type), 'Expected only types in the "output" list'); if (is_identifier($out_type)) { $out_type = lookupType($out_type->value()); } $name = 'val_' . $i; $my_output[$name] = $out_type; $i += 1; } } else { foreach ($output as $key => $out) { $name = $key; $my_output[$name] = $out; } } $debug = get_default($t_args, 'debug', 0); $simple = get_default($t_args, 'simple', false); $trimCR = get_default($t_args, 'trim.cr', false); // Handle separator $separator = ','; if (array_key_exists('sep', $t_args) || array_key_exists('separator', $t_args)) { $sep = get_first_key($t_args, ['sep', 'separator']); grokit_assert(is_string($sep), "Got " . gettype($sep) . " instead of string for separator."); if (strtolower($sep) === 'tab') { $sep = '\\t'; } grokit_assert($sep != "\n", 'CSV column delimiter cannot be new line'); // Scream if separator is longer than one character grokit_assert(\strlen($sep) == 1 || $sep == '\\t', 'Expected string of length 1 for separator, got string <' . $sep . '> instead'); $separator = $sep; } // Handle quote character $quotechar = '"'; if (array_key_exists('quote', $t_args) && !is_null($t_args['quote'])) { grokit_assert(!$simple, 'Quote option not available for simple CSVReader'); $quote = $t_args['quote']; grokit_assert(is_string($quote), "Got " . gettype($quote) . " instead of string for quote."); // Scream if separator is longer than one character grokit_assert(\strlen($quote) == 1, 'Expected string of length 1 for quote character, got string <' . $quote . '> instead'); $quotechar = $quote; } $quotechar = addcslashes($quotechar, '\\\''); // Handle escape character $escapeChar = '\\'; if (array_key_exists('escape', $t_args) && !is_null($t_args['escape'])) { grokit_assert(!$simple, 'Escape option not available for simple CSVReader'); $escape = $t_args['escape']; grokit_assert(is_string($escape), 'Got ' . gettype($escape) . ' instead of string for escape character.'); grokit_assert(\strlen($escape) == 1, 'Expected string of length 1 for escape character, got string <' . $escape . '> instead'); $escapeChar = $escape; } $escapeChar = addcslashes($escapeChar, '\\\''); // Handle header lines $headerLines = 0; if (array_key_exists('skip', $t_args)) { $headerLines = $t_args['skip']; grokit_assert(is_int($headerLines), 'Got ' . gettype($headerLines) . ' instead of int for number of lines to skip.'); grokit_assert($headerLines >= 0, 'Cannot skip a negative number of lines.'); } // Maximum number of lines to read $maxLines = get_default($t_args, 'n', -1); grokit_assert(is_int($maxLines), 'Got ' . gettype($maxLines) . ' instead of int for template argument "n"'); $nullArg = get_first_key_default($t_args, ['nullable'], false); $nullable = []; $nullStr = []; foreach ($my_output as $name => $type) { $nullable[$name] = false; } if ($nullArg === true) { foreach ($my_output as $name => $type) { $nullable[$name] = true; $nullStr[$name] = 'NULL'; } } else { if (is_array($nullArg)) { foreach ($nullArg as $n => $v) { // If nullable value is an associative mapping, the value is either true/false // or the value of the null string if (is_string($n)) { grokit_assert(is_string($v) || is_bool($v), 'CSVReader: nullable associative mapping must have string or boolean values'); grokit_assert(array_key_exists($n, $nullable), 'CSVReader: cannot make unknown attribute ' . $n . ' nullable'); if (is_bool($v)) { $nullable[$n] = $v; $nullStr[$n] = 'NULL'; } else { $nullable[$n] = true; $nullStr[$n] = $v; } } else { if (is_array($v)) { grokit_assert(array_key_exists('attr', $v), 'CSVReader: Name of nullable attribute not specified'); $attrName = $v['attr']->name(); $nullable[$attrName] = true; $nullStr[$attrName] = array_key_exists('null', $v) ? $v['null'] : 'NULL'; } else { // Otherwise, it's just nullable $attrName = $v->name(); grokit_assert(array_key_exists($attrName, $nullable), 'CSVReader: cannot make unknown attribute ' . $v . ' nullable'); $nullable[$attrName] = true; $nullStr[$attrName] = 'NULL'; } } } } else { if ($nullArg === false) { // Nothing } else { if (is_string($nullArg)) { foreach ($my_output as $name => $type) { $nullable[$name] = true; $nullStr[$name] = $nullArg; } } else { grokit_error('Template argument "nullable" must be boolean or array, ' . typeof($nullArg) . ' given'); } } } } // Come up with a name for ourselves $className = generate_name('CSVReader'); if ($debug >= 2) { foreach ($my_output as $name => $type) { fwrite(STDERR, "CSVReader: {$name} is nullable: " . ($nullable[$name] ? 'true' : 'false') . PHP_EOL); } } ?> class <?php echo $className; ?> { std::istream& my_stream; std::string fileName; // Template parameters static constexpr size_t MAX_LINES = <?php echo $maxLines; ?> ; static constexpr size_t HEADER_LINES = <?php echo $headerLines; ?> ; static constexpr char DELIMITER = '<?php echo $separator; ?> '; <?php if (!$simple) { ?> static constexpr char QUOTE_CHAR = '<?php echo $quotechar; ?> '; static constexpr char ESCAPE_CHAR = '<?php echo $escapeChar; ?> '; typedef boost::escaped_list_separator<char> separator; typedef boost::tokenizer< separator > Tokenizer; separator my_separator; Tokenizer my_tokenizer; <?php } ?> // Prevent having to allocate this every time. std::string line; std::vector<std::string> tokens; size_t count; <?php \grokit\declareDictionaries($my_output); ?> public: <?php echo $className; ?> ( GIStreamProxy& _stream ) : my_stream(_stream.get_stream()) , fileName(_stream.get_file_name()) <?php if (!$simple) { ?> , my_separator(ESCAPE_CHAR, DELIMITER, QUOTE_CHAR) , my_tokenizer(std::string("")) <?php } ?> , count(0) { <?php if ($headerLines > 0) { ?> for( size_t i = 0; i < HEADER_LINES; ++i ) { FATALIF( !getline( my_stream, line ), "CSV Reader reached end of file before finishing header.\n" ); } <?php } // If headerLines > 0 ?> } // > bool ProduceTuple( <?php echo typed_ref_args($my_output); ?> ) { if (count < MAX_LINES) { //> count++; } else { return false; } if( getline( my_stream, line ) ) { <?php if ($trimCR) { ?> if( line.back() == '\r' ) { line.pop_back(); } <?php } // if trimCR if (!$simple) { if ($debug >= 1) { ?> try { <?php } // if debug >= 1 ?> my_tokenizer.assign( line, my_separator ); <?php if ($debug >= 1) { ?> } catch(...) { FATAL("CSVReader for file %s failed on line: %s", fileName.c_str(), line.c_str()); } <?php } // if debug >= 1 ?> Tokenizer::iterator it = my_tokenizer.begin(); <?php foreach ($my_output as $name => $type) { if ($nullable[$name]) { // nullable ?> <?php \grokit\fromStringNullable($name, $type, 'it->c_str()', true, $nullStr[$name]); ?> <?php } else { // not nullable ?> <?php echo \grokit\fromStringDict($name, $type, 'it->c_str()'); ?> ; <?php } // end nullable check ?> ++it; <?php } // foreach output } else { ?> for( char & c : line ) { if( c == DELIMITER ) c = '\0'; } const char * ptr = line.c_str(); <?php $first = true; foreach ($my_output as $name => $type) { if ($first) { $first = false; } else { ?> while( *(ptr++) != '\0' ) ; // Advance past next delimiter <?php } // not first output if ($nullable[$name]) { ?> <?php echo \grokit\fromStringNullable($name, $type, 'ptr', true, $nullStr[$name]); } else { // not nullable ?> <?php echo \grokit\fromStringDict($name, $type, 'ptr'); ?> ; <?php } // if nullable } // foreach output } // if simple reader ?> return true; } else { return false; } } <?php \grokit\declareDictionaryGetters($my_output); ?> }; <?php $sys_headers = ['vector', 'string', 'iostream', 'cstdint']; if (!$simple) { $sys_headers[] = 'boost/tokenizer.hpp'; } return ['name' => $className, 'kind' => 'GI', 'output' => $my_output, 'system_headers' => $sys_headers, 'user_headers' => ['GIStreamInfo.h', 'Dictionary.h', 'DictionaryManager.h']]; }