function GroupBy(array $t_args, array $inputs, array $outputs, array $states) { // Ensure we have valid inputs. if (\count($inputs) == 0) { // No inputs given, try to get them from template arguments. grokit_assert(array_key_exists('input', $t_args), 'No inputs given for GroupBy'); $inputs = $t_args['input']; if (!is_array($inputs)) { $inputs = [$inputs]; } foreach ($inputs as $name => &$type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Invalid type given for input ' . $name); } } grokit_assert(array_key_exists('group', $t_args), 'No groups specified for GroupBy'); $gbyAttMap = $t_args['group']; grokit_assert(is_array($gbyAttMap), 'Invalid value given for groups, expected an expression name or list of expression names'); $gbyAttMap = array_map('strval', $gbyAttMap); $gbyAttNames = array_keys($gbyAttMap); foreach ($gbyAttMap as $in => $out) { grokit_assert(array_key_exists($in, $inputs), 'Group ' . $in . ' not present in input'); grokit_assert(array_key_exists($out, $outputs), 'Output Attribute ' . $out . ' for group ' . $in . ' not found in outputs'); } $numGByAtts = \count($gbyAttNames); grokit_assert(array_key_exists('aggregate', $t_args), 'No aggregate specified for GroupBy'); $innerGLA = $t_args['aggregate']; grokit_assert(is_gla($innerGLA), 'Non-GLA specified as aggregate for GroupBy'); $debug = get_default($t_args, 'debug', 0); $init_size = get_default($t_args, 'init.size', 1024); $use_mct = get_default($t_args, 'use.mct', true); $keepHashes = get_default($t_args, 'mct.keep.hashes', false); grokit_assert(is_bool($keepHashes), 'GroupBy mct.keep.hashes argument must be boolean'); // determine the result type $use_fragments = get_default($t_args, 'use.fragments', true); $resType = $use_fragments ? ['fragment', 'multi'] : ['multi']; $fragSize = get_default($t_args, 'fragment.size', 2000000); // Always support state $resType[] = 'state'; // Class name randomly generated $className = generate_name("GroupBy"); // instantiate the inner GLA. input/output is derived from the main input/output $gbyAtts = []; $gbyAttsOut = []; $glaInputAtts = []; $glaOutputAtts = []; foreach ($inputs as $name => $type) { if (in_array($name, $gbyAttNames)) { $gbyAtts[$name] = $type; $gbyAttsOut[$gbyAttMap[$name]] = $type; $outputs[$gbyAttMap[$name]] = $type; } else { $glaInputAtts[$name] = $type; } } foreach ($outputs as $name => $type) { if (!in_array($name, $gbyAttMap)) { $glaOutputAtts[$name] = $type; } } $innerGLA = $innerGLA->apply($glaInputAtts, $glaOutputAtts, $states); $libraries = $innerGLA->libraries(); $innerRes = get_first_value($innerGLA->result_type(), ['multi', 'single', 'state']); if ($innerRes == 'state') { // If the result type is state, the only output is a state object // containing the GLA. $outputName = array_keys($glaOutputAtts)[0]; $innerOutputs = [$outputName => lookupType('base::STATE', ['type' => $innerGLA])]; } else { $innerOutputs = $innerGLA->output(); grokit_assert(\count($innerOutputs) == \count($glaOutputAtts), 'Expected ' . \count($glaOutputAtts) . ' outputs fromm Inner GLA, got ' . \count($innerOutputs)); } $constState = lookupResource('GroupByState', ['gla' => $innerGLA, 'groups' => $gbyAtts, 'debug' => $debug]); // constructor argumetns are inherited from inner GLA $configurable = $innerGLA->configurable(); $reqStates = $innerGLA->req_states(); // We need to specially create the constructor string because apparently // declaring Type Name(); is a function declaration instead of a variable // declaration for some reason. $constructorParts = []; if ($configurable) { $constructorParts[] = 'jsonInit'; } if ($innerGLA->has_state()) { $constructorParts[] = 'innerState'; } $constructorString = \count($constructorParts) > 0 ? '(' . implode(', ', $constructorParts) . ')' : ''; // add the outputs we got from the gla foreach ($innerOutputs as $name => $type) { grokit_assert(array_key_exists($name, $outputs), 'Inner GLA\'s outputs refer to unknown attribute ' . $name); grokit_assert($type !== null, 'GroupBy Inner GLA left output ' . $name . ' with no type'); $outputs[$name] = $type; } $iterable = $innerGLA->iterable(); // need to keep track of system includes needed $extraHeaders = array(); $allocatorText = "std::allocator<std::pair<const Key, {$innerGLA}> >"; if ($use_mct) { $keepHashesText = $keepHashes ? 'true' : 'false'; $extraHeaders[] = "mct/hash-map.hpp"; $map = "mct::closed_hash_map<Key, {$innerGLA}, HashKey, std::equal_to<Key>, {$allocatorText}, {$keepHashesText}>"; $mapType = 'mct::closed_hash_map'; } else { $extraHeaders[] = "unordered_map"; $map = "std::unordered_map<Key, {$innerGLA}, HashKey, std::equal_to<Key>, {$allocatorText}>"; $mapType = 'std::unordered_map'; } if ($debug > 0) { $extraHeaders[] = 'cstdio'; } ?> class <?php echo $className; ?> { public: using ConstantState = <?php echo $constState; ?> ; <?php if ($innerGLA->has_state()) { ?> using InnerState = ConstantState::InnerState; <?php } // if gla has state ?> using Key = ConstantState::Key; using HashKey = ConstantState::HashKey; using InnerGLA = <?php echo $innerGLA; ?> ; typedef <?php echo $map; ?> MapType; static const size_t INIT_SIZE = <?php echo $init_size; ?> ; public: class Iterator { MapType::iterator it; // current value MapType::iterator end; // last value in the fragment public: Iterator() { } Iterator(MapType::iterator _it, MapType::iterator _end): it(_it), end(_end) { if( it != end ) { <?php switch ($innerRes) { case 'multi': ?> it->second.Finalize(); <?php break; case 'state': if ($innerGLA->finalize_as_state()) { ?> it->second.FinalizeState(); <?php } // if we need to finalize as a state break; } // end switch inner restype ?> } } bool GetNextResult( <?php echo typed_ref_args($outputs); ?> ) { bool gotResult = false; while( it != end && !gotResult ) { <?php echo $innerGLA; ?> & gla = it->second; <?php foreach ($gbyAttMap as $in => $out) { ?> <?php echo $out; ?> = it->first.<?php echo $in; ?> ; <?php } // foreach grouping attribute ?> <?php switch ($innerRes) { case 'multi': ?> gotResult = gla.GetNextResult( <?php echo args($innerOutputs); ?> ); if( !gotResult ) { ++it; if( it != end ) { it->second.Finalize(); } } <?php break; case 'single': ?> gotResult = true; gla.GetResult(<?php echo args($innerOutputs); ?> ); ++it; <?php break; case 'state': reset($innerOutputs); // Assuming that $innerOutputs contains a single value that is // the state type. $oName = key($innerOutputs); $oType = current($innerOutputs); ?> gotResult = true; <?php echo $oName; ?> = <?php echo $oType; ?> ( &gla ); ++it; <?php } // switch inner result type ?> } return gotResult; } }; private: const ConstantState & constState; <?php if ($configurable) { ?> const Json::Value jsonInit; <?php } // if configurable ?> size_t count; MapType groupByMap; std::vector<MapType::iterator> theIterators; // the iterators, only 2 elements if multi, many if fragment Iterator multiIterator; public: <?php echo $className; ?> (<?php if ($configurable) { ?> const Json::Value & _jsonInit, <?php } ?> const ConstantState & _constState ) : constState(_constState) <?php if ($configurable) { ?> , jsonInit(_jsonInit) <?php } // if configurable ?> , count(0) , groupByMap( INIT_SIZE ) , theIterators() , multiIterator() { } ~<?php echo $className; ?> () {} void Reset(void) { count = 0; groupByMap.clear(); theIterators.clear(); } void AddItem(<?php echo array_template('const {val} & {key}', ', ', $inputs); ?> ) { count++; // check if _key is already in the map; if yes, add _value; else, add a new // entry (_key, _value) Key key(<?php echo array_template('{key}', ', ', $gbyAtts); ?> ); MapType::iterator it = groupByMap.find(key); if (it == groupByMap.end()) { // group does not exist // create an empty GLA and insert // better to not add the item here so we do not have // to transport a large state <?php if ($innerGLA->has_state()) { ?> const InnerState & innerState = constState.getConstState(key); <?php } // if gla has state ?> InnerGLA gla<?php echo $constructorString; ?> ; auto ret = groupByMap.insert(MapType::value_type(key, gla)); it = ret.first; // reposition } it->second.AddItem(<?php echo array_template('{key}', ', ', $glaInputAtts); ?> ); } void AddState(<?php echo $className; ?> & other) { count += other.count; // scan other hash and insert or update content in this one for (MapType::iterator it = other.groupByMap.begin(); it != other.groupByMap.end(); ++it) { const Key& okey = it->first; <?php echo $innerGLA; ?> & ogla = it->second; MapType::iterator itt = groupByMap.find(okey); if (itt != groupByMap.end()) { // found the group <?php echo $innerGLA; ?> & gla = itt->second; gla.AddState(ogla); } else { // add the other group to this hash groupByMap.insert(MapType::value_type(okey, ogla)); } } } <?php if ($iterable) { ?> bool ShouldIterate(ConstantState& modibleState) { <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : ==== ShouldIterate ====\n"); <?php } // if debugging enabled ?> bool shouldIterate = false; for( MapType::iterator it = groupByMap.begin(); it != groupByMap.end(); ++it ) { const Key & key = it->first; InnerGLA & gla = it->second; <?php if ($innerGLA->has_state()) { ?> InnerState & innerState = modibleState.getModibleState(key); <?php } // if gla has state ?> bool glaRet = gla.ShouldIterate(innerState); shouldIterate = shouldIterate || glaRet; <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : Key(%s) shouldIterate(%s)\n", key.to_string().c_str(), glaRet ? "true" : "false"); <?php } // if debugging enabled ?> } return shouldIterate; } <?php } // if iterable ?> <?php if (in_array('fragment', $resType)) { ?> int GetNumFragments(void){ int size = groupByMap.size(); int sizeFrag = <?php echo $fragSize; ?> ; // setup the fragment boundaries // scan via iterator and count int frag=0; int pos=0; MapType::iterator it = groupByMap.begin(); theIterators.clear(); theIterators.push_back( it ); // special case when size < num_fragments // > if (sizeFrag == 0){ it = groupByMap.end(); theIterators.push_back( it ); return 1; // one fragment } while(it!=groupByMap.end()){ while(it!=groupByMap.end() && pos<( frag + 1 )*sizeFrag){ //> ++it; pos++; } theIterators.push_back( it ); frag++; } <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : fragments(%d)\n", frag); <?php } ?> return frag; } Iterator* Finalize(int fragment){ // Call finalize on all inner GLAs in this fragment. MapType::iterator iter = theIterators[fragment]; MapType::iterator iterEnd = theIterators[fragment+1]; Iterator* rez = new Iterator(theIterators[fragment], theIterators[fragment+1] ); return rez; } bool GetNextResult(Iterator* it, <?php echo array_template('{val} & {key}', ', ', $outputs); ?> ) { return it->GetNextResult(<?php echo args($outputs); ?> ); } <?php } // if using fragment interface ?> void Finalize() { multiIterator = Iterator( groupByMap.begin(), groupByMap.end() ); <?php if ($debug >= 1) { ?> fprintf(stderr, "<?php echo $className; ?> : groups(%lu) tuples(%lu)\n", groupByMap.size(), count); <?php } ?> } bool GetNextResult(<?php echo array_template('{val} & {key}', ', ', $outputs); ?> ) { return multiIterator.GetNextResult( <?php echo args($outputs); ?> ); } std::size_t size() const { return groupByMap.size(); } const MapType& GetMap() const { return groupByMap; } bool Contains(<?php echo const_typed_ref_args($gbyAtts); ?> ) const { Key key(<?php echo args($gbyAtts); ?> ); return groupByMap.count(key) > 0; } const InnerGLA& Get(<?php echo const_typed_ref_args($gbyAtts); ?> ) const { Key key(<?php echo args($gbyAtts); ?> ); return groupByMap.at(key); } bool Contains(Key key) const { return groupByMap.count(key) > 0; } const InnerGLA& Get(Key key) const { return groupByMap.at(key); } }; <?php if (in_array('fragment', $resType)) { ?> typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php } ?> <?php $sys_headers = array_merge(['iomanip', 'iostream', 'cstring'], $extraHeaders); return array('kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => array('HashFunctions.h'), 'input' => $inputs, 'output' => $outputs, 'result_type' => $resType, 'configurable' => $configurable, 'generated_state' => $constState, 'required_states' => $reqStates, 'iterable' => $iterable, 'properties' => ['resettable', 'finite container'], 'libraries' => $libraries, 'extra' => ['inner_gla' => $innerGLA]); }
function STATE(array $t_args) { $type = get_first_key($t_args, ['type', '0']); grokit_assert(is_gla($type), 'Template argument to STATE must be a valid GLA.'); $type = $type->lookup(); $gContent = ''; $functions = []; $methods = []; $className = generate_name('STATE_'); ?> /** Type definition for generic GLA states. This type is only used to trannsport states withing the same memory space between operators. The object the state points to MUST be treated like a const. Note: this type cannot be read from the disk or written to the output. A different mechanism will be used for that. The type in the object must be a hash of the name of the class used to encode the object. Any function that assumes a certain type must explicitly verify the correctness of the type. The object can be manipulated like a basic datatype. STATE objects do not know how to deallocate the memory they use. Other mechanisms have to be used to ensure correct deallocation (acknowledgements of data packets that contain this as members). **/ class <?php echo $className; ?> { public: typedef <?php echo $type; ?> * pointer_type; typedef uint64_t hash_type; private: pointer_type object; hash_type type; public: <?php echo $className; ?> (): object(nullptr), type(0) {} <?php echo $className; ?> (pointer_type _object): object(_object), type(<?php echo $type->cHash(); ?> ) {} pointer_type GetObject() const { FATALIF(type != <?php echo $type->cHash(); ?> , "STATE contains incorrect type!"); return object; } <?php $methods[] = ['IsNull', [], 'BASE::BOOL', true]; ?> bool IsNull() const { return object == nullptr; } /** no destructor. object should not be deallocated here */ }; <?php ob_start(); ?> <?php $functions[] = ['IsNull', ['@type'], 'BASE::BOOL', true, true]; ?> inline bool IsNull( const @type & d ) { return d.IsNull(); } <?php $gContent .= ob_get_clean(); ?> <?php return array('kind' => 'TYPE', 'name' => $className, "complex" => false, 'extras' => ['type' => $type], 'properties' => ['__state__'], 'global_content' => $gContent, 'methods' => $methods, 'functions' => $functions); }
function Multiplexer(array $t_args, array $inputs, array $outputs) { $className = generate_name('Multiplexer'); if (\count($inputs) == 0) { grokit_assert(array_key_exists('input', $t_args), 'No inputs specified for Multiplexer'); $inputs = $t_args['input']; foreach ($t_args['inputs'] as $name => &$type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Only types may be specified as inputs to Multiplexer.'); } $inputs = ensure_valid_names($inputs, 'multi_input'); } $glas = get_first_key($t_args, ['glas', 0]); grokit_assert(\count($glas) > 0, 'No GLAs specified for Multiplexer.'); $myGLAs = []; $glaInputs = []; $glaOutputs = []; $resultType = 'multi'; $usedOutputs = []; $libraries = []; $glaGenStates = []; $glaReqStates = []; $configurable = false; $constArgs = []; $genStates = []; $reqStates = []; $iterable = null; foreach ($glas as $name => $glaInfo) { grokit_assert(is_array($glaInfo), 'Template argument \'glas\' must be an array'); grokit_assert(array_key_exists('gla', $glaInfo), 'No GLA given for glas[' . $name . ']'); grokit_assert(array_key_exists('inputs', $glaInfo), 'No inputs given for glas[' . $name . ']'); grokit_assert(array_key_exists('outputs', $glaInfo), 'No outputs given for glas[' . $name . ']'); $gla = $glaInfo['gla']; $glaInAtts = $glaInfo['inputs']; $glaOutAtts = $glaInfo['outputs']; grokit_assert(is_gla($gla), 'Non-GLA given for glas[' . $name . '][gla]'); grokit_assert(is_array($glaInAtts), 'Non-array given for inputs for gla ' . $name); grokit_assert(is_array($glaOutAtts), 'Non-array given for outputs for gla ' . $name); $glaInAtts = array_map('strval', $glaInAtts); $glaOutAtts = array_map('strval', $glaOutAtts); $glaName = "innerGLA_" . $name; $glaInputs[$glaName] = []; $glaOutputs[$glaName] = []; foreach ($glaInAtts as $att) { grokit_assert(array_key_exists($att, $inputs), 'Input ' . $att . ' for GLA ' . $name . ' not found in inputs'); $glaInputs[$glaName][$att] = $inputs[$att]; } foreach ($glaOutAtts as $att) { grokit_assert(array_key_exists($att, $outputs), 'Output ' . $att . ' for GLA ' . $name . ' not found in outputs'); grokit_assert(!in_array($att, $usedOutputs), 'Output ' . $att . ' used by multiple GLAs'); $usedOutputs[] = $att; $glaOutputs[$glaName][$att] = $outputs[$att]; } //fwrite(STDERR, "Inputs for GLA " . $glaName . ": " . print_r($glaInputs[$glaName], true) . PHP_EOL ); //fwrite(STDERR, "Outputs for GLA " . $glaName . ": " . print_r($glaOutputs[$glaName], true) . PHP_EOL ); $gla = $gla->apply($glaInputs[$glaName], $glaOutputs[$glaName]); $myGLAs[$glaName] = $gla; $glaRez[$glaName] = get_first_value($gla->result_type(), ['multi', 'single', 'state']); $libraries = array_merge($libraries, $gla->libraries()); if ($glaRez[$glaName] == 'state') { grokit_assert(\count($glaOutputs[$glaName]) == 1, "GLA {$glaName} is produced as state, and thus must have exactly 1 output."); $stateType = lookupType('base::STATE', ['type' => $gla]); $glaOutputs[$glaName] = array_combine(array_keys($glaOutputs[$glaName]), [$stateType]); } else { grokit_assert(\count($glaOutputs[$glaName]) == \count($gla->output()), 'GLA ' . $glaName . ' produces different number of outputs than expected'); $glaOutputs[$glaName] = array_combine(array_keys($glaOutputs[$glaName]), $gla->output()); } // Set types for our output foreach ($glaOutputs[$glaName] as $attName => $type) { $outputs[$attName] = $type; } if (is_null($iterable)) { $iterable = $gla->iterable(); } else { grokit_assert($iterable == $gla->iterable(), 'Multiplexer does not support mixing iterable and non-iterable GLAs'); } $glaReqStates[$glaName] = $gla->req_states(); foreach ($gla->req_states() as $rstate) { $reqStates[] = $rstate; } $glaGenStates[$glaName] = $gla->state(); // TODO: Support constant states grokit_assert(!$gla->has_state(), 'Multiplexer currently does not support constant states.'); } $libraries = array_unique($libraries); $extra = ['glas' => $myGLAs]; ?> class <?php echo $className; ?> { <?php foreach ($myGLAs as $name => $type) { ?> <?php echo $type; ?> <?php echo $name; ?> ; <?php } // foreach inner gla ?> class Iterator { bool _gotResultsOnce; bool _valid; <?php foreach ($myGLAs as $name => $type) { ?> <?php echo $type; ?> * it_<?php echo $name; ?> ; <?php } // foreach inner gla ?> public: Iterator(void) : _gotResultsOnce(false), _valid(false), <?php echo array_template('it_{key}(nullptr)', ', ', $myGLAs); ?> { } Iterator(<?php echo typed_ref_args($myGLAs); ?> ) : _gotResultsOnce(false), _valid(true), <?php echo array_template('it_{key}(&{key})', ', ', $myGLAs); ?> { <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'multi') { ?> <?php echo $name; ?> .Finalize(); <?php } // if inner GLA is multi } // foreach inner gla ?> } Iterator( const Iterator & other) = default; ~Iterator() { } bool GetNextResult( <?php echo typed_ref_args($outputs); ?> ) { FATALIF(!_valid, "Tried to get results from an invalid iterator."); bool ret = !_gotResultsOnce; _gotResultsOnce = true; <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'multi') { ?> ret |= it_<?php echo $name; ?> ->GetNextResult(<?php echo args($glaOutputs[$name]); ?> ); <?php } // if inner GLA is multi } // foreach inner gla ?> if( ret ) { <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'single') { ?> it_<?php echo $name; ?> ->GetResult(<?php echo args($glaOutputs[$name]); ?> ); <?php } else { if ($glaRez[$name] == 'state') { $stateVar = array_keys($glaOutputs[$name])[0]; $stateType = $glaOutputs[$name][$stateVar]; ?> <?php echo $stateVar; ?> = <?php echo $stateType; ?> (it_<?php echo $name; ?> ); <?php } } // if inner GLA is state } // foreach inner gla ?> } return ret; } }; Iterator multiIterator; public: <?php echo $className; ?> () { } ~<?php echo $className; ?> () { } void AddItem(<?php echo const_typed_ref_args($inputs); ?> ) { // Call AddItem individually on each GLA. <?php foreach ($myGLAs as $gName => $gType) { ?> <?php echo $gName; ?> .AddItem(<?php echo args($glaInputs[$gName]); ?> ); <?php } // foreach inner gla ?> } void AddState( <?php echo $className; ?> & other ) { // Call AddState individually on each GLA. <?php foreach ($myGLAs as $gName => $gType) { ?> <?php echo $gName; ?> .AddState(other.<?php echo $gName; ?> ); <?php } // foreach inner gla ?> } void Finalize() { multiIterator = Iterator(<?php echo args($myGLAs); ?> ); } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { return multiIterator.GetNextResult(<?php echo args($outputs); ?> ); } void GetResult(<?php echo typed_ref_args($outputs); ?> ) { Finalize(); GetNextResult(<?php echo args($outputs); ?> ); } <?php foreach (array_keys($myGLAs) as $index => $name) { ?> const <?php echo $myGLAs[$name]; ?> & GetGLA<?php echo $index; ?> () const { return <?php echo $name; ?> ; } <?php } ?> }; <?php return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => $resultType, 'libraries' => $libraries, 'configurable' => $configurable, 'extra' => $extra); }
function hashComplex($val) { $hasher = hash_init('sha256'); if (is_array($val)) { hash_update($hasher, '['); // Ensure array is sorted by keys ksort($val); foreach ($val as $name => $v) { hash_update($hasher, $name); hash_update($hasher, '=>'); hash_update($hasher, hashComplex($v)); } hash_update($hasher, ']'); } else { if (is_gla($val)) { hash_update($hasher, 'gla'); hash_update($hasher, $val->hash()); } else { if (is_gf($val)) { hash_update($hasher, 'gf'); hash_update($hasher, $val->hash()); } else { if (is_gt($val)) { hash_update($hasher, 'gt'); hash_update($hasher, $val->hash()); } else { if (is_gist($val)) { hash_update($hasher, 'gist'); hash_update($hasher, $val->hash()); } else { if (is_gi($val)) { hash_update($hasher, 'gi'); hash_update($hasher, $val->hash()); } else { if (is_datatype($val)) { hash_update($hasher, 'datatype'); hash_update($hasher, $val->hash()); } else { if (is_functor($val)) { hash_update($hasher, 'functor'); hash_update($hasher, $val->hash()); } else { if (is_identifier($val)) { hash_update($hasher, 'identifier'); hash_update($hasher, $val->hash()); } else { if (is_attribute($val)) { hash_update($hasher, 'attribute'); hash_update($hasher, strval($val)); } else { if (is_int($val) || is_float($val) || is_string($val) || is_bool($val)) { hash_update($hasher, gettype($val)); hash_update($hasher, $val); } else { if (is_null($val)) { hash_update($hasher, 'null'); } else { $valType = is_object($val) ? get_class($val) : gettype($val); grokit_logic_error('Unable to hash unknown type ' . $valType); } } } } } } } } } } } } return hash_final($hasher); }
function Segmenter(array $t_args, array $input, array $output, array $given_states) { $resType = ['fragment', 'multi']; $system_headers = ['array', 'vector', 'memory', 'cinttypes', 'unordered_map']; $user_headers = ['HashFunctions.h']; $lib_headers = []; $preferFragment = get_default($t_args, 'inner.prefer.fragment', false); $wantedRes = $preferFragment ? ['fragment', 'multi'] : ['multi', 'fragment']; $nInputs = \count($input); grokit_assert($nInputs > 1, 'Segmenter: Not enough inputs specified!'); $keyName = array_keys($input)[0]; $keyType = array_get_index($input, 0); $innerInputs = array_slice($input, 1, $nInputs - 1, true); $gla = get_first_key($t_args, ['gla', 'GLA', 0]); grokit_assert(is_gla($gla), 'Segmenter: [gla] argument must be a valid GLA'); $gla = $gla->apply($innerInputs, $output, $given_states); $n_passes = get_default($t_args, 'passes', 1); grokit_assert(is_int($n_passes), 'Segmenter: [passes] argument must be an integer'); grokit_assert($n_passes > 0, 'Segmenter: [passes] argument must be > 0'); $libraries = $gla->libraries(); $innerRes = get_first_value($gla->result_type(), $wantedRes); $innerInputs = $gla->input(); $innerOutput = $gla->output(); $input = array_merge([$keyName => $keyType], $innerInputs); $output = $innerOutput; $segments = get_default($t_args, 'segments', 64); $constState = lookupResource('BASE::SegmenterState', ['gla' => $gla, 'passes' => $n_passes, 'segments' => $segments]); $className = generate_name('Segmenter_'); $savedArgs = []; $cArgs = []; $innerCArgs = []; if ($gla->configurable()) { $savedArgs['json_init'] = 'Json::Value'; $cArgs['json_init'] = 'Json::Value'; $innerCArgs[] = 'json_init'; } $cArgs['const_state'] = $constState; if ($gla->has_state()) { $innerCArgs[] = 'constState.inner_cstate'; } $cstStr = \count($innerCArgs) > 0 ? '(' . implode(',', $innerCArgs) . ')' : ''; grokit_assert(!$gla->iterable(), 'Segementer does not support iterable GLAs'); $iterable = $n_passes > 1; ?> class <?php echo $className; ?> { private: using ConstantState = <?php echo $constState; ?> ; using SplitState = ConstantState::SplitState; static constexpr const size_t NUM_STATES = SplitState::NUM_STATES; using InnerGLA = <?php echo $gla; ?> ; using InnerGLAPtr = std::unique_ptr<InnerGLA>; using GLA_Array = std::array<InnerGLAPtr, NUM_STATES>; public: using size_type = std::size_t; <?php if ($innerRes == 'fragment') { ?> class Iterator { private: InnerGLA * gla; int fragmentNum; InnerGLA::Iterator * innerIter; public: Iterator( InnerGLA * _gla, int _fragmentNum, int _innerFrag ) : gla(_gla), fragmentNum(_fragmentNum), innerIter(nullptr) { innerIter = gla->Finalize(_innerFrag); } ~Iterator(void) { if( innerIter != nullptr ) { delete innerIter; innerIter = nullptr; } } bool GetNextResult( <?php echo typed_ref_args($gla->output()); ?> ) { return innerIter->GetNextResult(<?php echo args($gla->output()); ?> ); } int FragmentNumber() { return fragmentNum; } }; <?php } else { // if inner result type is fragment ?> class Iterator { private: InnerGLA * gla; int fragmentNum; public: Iterator( InnerGLA * _gla, int fragNo ) : gla(_gla), fragmentNum(fragNo) { gla->Finalize(); } ~Iterator(void) { } bool GetNextResult( <?php echo typed_ref_args($gla->output()); ?> ) { return gla->GetNextResult(<?php echo args($gla->output()); ?> ); } int FragmentNumber() { return fragmentNum; } }; <?php } // if inner result type is multi ?> private: const ConstantState & constState; GLA_Array localState; // Iteration state for multi result type int numFrags; int multiFragNo; Iterator * multiIter; <?php if ($innerRes == 'fragment') { ?> using frag_info = std::pair<int, int>; using frag_map_t = std::unordered_map<int, frag_info>; frag_map_t fragMap; <?php } ?> <?php foreach ($savedArgs as $name => $type) { ?> const <?php echo $type; ?> <?php echo $name; ?> ; <?php } // foreach saved arg ?> public: // Constructor <?php echo $className; ?> ( <?php echo const_typed_ref_args($cArgs); ?> ) : constState(const_state) , localState() , numFrags(0) , multiFragNo(0) , multiIter(nullptr) <?php if ($innerRes == 'fragment') { ?> , fragMap() <?php } foreach ($savedArgs as $name => $type) { ?> , <?php echo $name; ?> (<?php echo $name; ?> ) <?php } // foreach constructor arg to save ?> { for( auto & elem : localState ) { elem.reset(new InnerGLA<?php echo $cstStr; ?> ); } } void AddItem( <?php echo const_typed_ref_args($input); ?> ) { uint64_t hashVal = CongruentHash(Hash(<?php echo $keyName; ?> ), H_b + 1); uint64_t passNum = (hashVal / NUM_STATES) % ConstantState::N_PASSES; uint64_t segNum = hashVal % NUM_STATES; <?php if ($n_passes > 1) { ?> if( passNum != constState.pass ) { return; } <?php } // more than 1 pass ?> localState[segNum]->AddItem(<?php echo args($innerInputs); ?> ); } void ChunkBoundary(void) { // Merge local states into the global state SplitState & globalStates = constState.segments; int theseAreOk[NUM_STATES]; for( int i = 0; NUM_STATES > i; i++ ) { theseAreOk[i] = 1; } int segsLeft = NUM_STATES; while( segsLeft > 0 ) { InnerGLA * checkedOut = nullptr; int whichOne = globalStates.CheckOutOne( theseAreOk, checkedOut ); if( checkedOut == NULL ) { checkedOut = new InnerGLA<?php echo $cstStr; ?> ; } checkedOut->AddState( *(localState[whichOne]) ); globalStates.CheckIn( whichOne, checkedOut ); theseAreOk[whichOne] = 0; segsLeft--; } // Re-initialize the local states for( auto & elem : localState ) { <?php if ($gla->is('resettable')) { ?> elem->Reset(); <?php } else { // if resettable ?> elem.reset(new InnerGLA<?php echo $cstStr; ?> ); <?php } // if not resettable ?> } } void AddState( <?php echo $className; ?> & o ) { // Do nothing } void Finalize() { SplitState & globalStates = constState.segments; if( multiIter != nullptr) delete multiIter; multiFragNo = 0; <?php if ($innerRes == 'fragment') { ?> frag_info fInfo = fragMap[multiFragNo]; multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo, fInfo.second); <?php } else { ?> multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo); <?php } ?> } bool GetNextResult(<?php echo typed_ref_args($output); ?> ) { bool gotResult = false; SplitState & globalStates = constState.segments; while( (multiFragNo < numFrags && multiIter != nullptr) && !gotResult ) { gotResult = multiIter->GetNextResult(<?php echo args($output); ?> ); if( !gotResult ) { multiFragNo++; delete multiIter; if( numFrags > multiFragNo ) { <?php if ($innerRes == 'fragment') { ?> frag_info fInfo = fragMap[multiFragNo]; multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo, fInfo.second); <?php } else { ?> multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo); <?php } ?> } else { multiIter = nullptr; } } } return gotResult; } int GetNumFragments(void) { <?php if ($innerRes == 'fragment') { ?> SplitState & globalStates = constState.segments; numFrags = 0; for (int i = 0; i < NUM_STATES; i++) { int curFrags = globalStates.Peek(i)->GetNumFragments(); for (int curFrag = 0; curFrag < curFrags; curFrag++) { fragMap[numFrags] = frag_info(i, curFrag); numFrags++; } } <?php } else { ?> numFrags = NUM_STATES; <?php } ?> return numFrags; } Iterator * Finalize( int fragment ) { SplitState & globalStates = constState.segments; <?php if ($innerRes == 'fragment') { ?> frag_info info = fragMap[fragment]; return new Iterator(globalStates.Peek(info.first), fragment, info.second); <?php } else { ?> return new Iterator(globalStates.Peek(fragment), fragment); <?php } ?> } bool GetNextResult( Iterator * it, <?php echo typed_ref_args($output); ?> ) { bool ret = it->GetNextResult(<?php echo args($output); ?> ); return ret; } <?php if ($iterable) { ?> bool ShouldIterate( ConstantState & modible ) { modible.pass++; return modible.pass < ConstantState::N_PASSES; } void PostFinalize() { constState.segments.Reset(); } <?php } // iterable ?> <?php if ($gla->is('finite container')) { ?> size_type size() { SplitState & globalStates = constState.segments; size_type s = 0; for( int i = 0; NUM_STATES > i; i++ ) { InnerGLA * ptr = globalStates.Peek(i); s += ptr->size(); } return s; } size_type size(int frag) { SplitState & globalStates = constState.segments; return globalStates.Peek(frag)->size(); } <?php } // if the gla is a container ?> }; typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $system_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'input' => $input, 'output' => $output, 'result_type' => $resType, 'generated_state' => $constState, 'required_states' => $gla->req_states(), 'chunk_boundary' => true, 'configurable' => $gla->configurable(), 'iterable' => $iterable, 'post_finalize' => $iterable, 'intermediates' => true]; }