function CreateView($args, $targs) { grokit_assert(\count($args) == 1, 'CreateView supports exactly 1 input'); $type = $args[0]; grokit_assert($type->is('array'), 'CreateView cannot create view on non-array type'); $innerType = $type->get('type'); $size = $type->get('size'); $viewType = lookupType('BASE::FixedArrayView', ['type' => $innerType, 'size' => $size]); $funcname = generate_name('CreateView_'); ?> <?php echo $viewType; ?> <?php echo $funcname; ?> ( const <?php echo $type; ?> &array ) { return <?php echo $viewType; ?> (array.data()); } <?php return ['kind' => 'FUNCTION', 'name' => $funcname, 'input' => $args, 'result' => $viewType, 'deterministic' => false]; }
function _defineStdMathFunction($name, $args, $ret, $lib) { $f = function () use($name, $args, $ret, $lib) { $inputs = []; $count = 0; foreach ($args as $arg) { $inputs['arg' . $count] = lookupType($arg); $count += 1; } $retType = lookupType($ret); $hash = \grokit\hashComplex(['BASE::' . $name, $args, $ret]); ?> inline <?php echo $retType; ?> <?php echo $name; ?> ( <?php echo typed_args($inputs); ?> ) { return std::<?php echo $name; ?> (<?php echo args($inputs); ?> ); } <?php return ['kind' => 'FUNCTION', 'name' => $name, 'input' => $inputs, 'result' => $retType, 'deterministic' => true, 'system_headers' => $lib, 'hash' => $hash]; }; declareFunction($name, $args, $f); }
function Hash($args, array $t_args = []) { $funcName = generate_name('Hash_'); $retType = lookupType('base::BIGINT'); echo $retType; ?> <?php echo $funcName; ?> ( <?php echo const_typed_ref_args($args); ?> ) { uint64_t hashVal = H_b; <?php foreach ($args as $name => $type) { ?> hashVal = CongruentHash(<?php echo $name; ?> , hashVal); <?php } // foreach argument ?> return static_cast<<?php echo $retType; ?> >(hashVal); } <?php return ['kind' => 'FUNCTION', 'name' => $funcName, 'result' => $retType, 'system_headers' => ['cinttypes'], 'user_headers' => ['HashFunctions.h'], 'deterministic' => true]; }
public function type() { if (!is_null($this->type)) { return lookupType($this->type->name(), $this->type->template_args()); } if (is_null($this->type_ast)) { return null; } return parseType($this->type_ast); }
function Contains($args, $targs) { grokit_assert(\count($args) == 1, 'Contains supports exactly 1 input, ' . \count($args) . ' given'); grokit_assert(array_key_exists('values', $targs), 'Contains() requires a "values" template argument'); $inputName = 'contains_input'; $inputType = $args[0]; $boolType = lookupType('base::bool'); $typename = generate_name('_ContainsType'); $funcname = generate_name('Contains'); $sys_headers = ['cstddef']; $use_mct = get_default($targs, 'use.mct', false); if ($use_mct) { $sys_headers[] = 'mct/closed-hash-set.hpp'; $setType = 'mct::closed_hash_set<' . $inputType . ', KeyHash>'; } else { $sys_headers[] = 'unordered_set'; $setType = 'std::unordered_set<' . $inputType . ', KeyHash>'; } $values = $targs['values']; grokit_assert(is_array($values), 'Contains(): values argument must be an array of strings'); $quotedValues = []; $escapeChars = "\"'\n\r\t\\"; foreach ($values as $index => $val) { grokit_assert(is_string($val), "Contains(): Value at index {$index} is not a string"); $quotedValues[] = '"' . addcslashes($val, $escapeChars) . '"'; } $nVals = \count($quotedValues); ?> class <?php echo $typename; ?> { public: struct KeyHash { std::size_t operator () (const <?php echo $inputType; ?> & val) const { return static_cast<std::size_t>(Hash(val)); } }; using Set = <?php echo $setType; ?> ; // Singleton static const <?php echo $typename; ?> instance; private: static const char* str_values[<?php echo $nVals; ?> ]; Set values; <?php echo $typename; ?> (): values() { <?php echo $inputType; ?> temp; for( auto str : str_values ) { FromString(temp, str); values.insert(temp); } } public: bool exists(const <?php echo $inputType; ?> & <?php echo $inputName; ?> ) const { return values.count(<?php echo $inputName; ?> ) > 0; } }; const <?php echo $typename; ?> <?php echo $typename; ?> ::instance; const char* <?php echo $typename; ?> ::str_values[<?php echo $nVals; ?> ] = { <?php echo implode(", ", $quotedValues); ?> }; <?php echo $boolType; ?> <?php echo $funcname; ?> (const <?php echo $inputType; ?> & <?php echo $inputName; ?> ) { return <?php echo $typename; ?> ::instance.exists(<?php echo $inputName; ?> ); } <?php return ['kind' => 'FUNCTION', 'name' => $funcname, 'input' => $args, 'result' => $boolType, 'determinstic' => true, 'system_headers' => $sys_headers]; }
/** * A GLA that estimates the cardinality of a dataset using a bloom filter of * a configurable size. * * Note: This filter has very high performance, so long as all of the states * fit into cache, preferably L1 or L2, but L3 is also fine. Once the states * are large enough that all of them cannot fit inside L3 cache at the same * time, performance takes a nose dive (4x loss minimum). */ function BloomFilter(array $t_args, array $input, array $output) { grokit_assert(\count($output) == 1, 'BloomFilter produces only 1 value, ' . \count($output) . ' outputs given.'); $outputName = array_keys($output)[0]; $outputType = array_get_index($output, 0); if (is_null($outputType)) { $outputType = lookupType('BASE::BIGINT'); } $output[$outputName] = $outputType; grokit_assert($outputType->is('numeric'), 'BloomFilter output must be numeric!'); $exp = get_first_key_default($t_args, ['exponent'], 16); grokit_assert(is_integer($exp), 'BloomFilter exponent must be an integer.'); grokit_assert($exp > 0 && $exp < 64, 'BloomFilter exponent must be in range (0,64), ' . $exp . ' given.'); $nullCheck = get_default($t_args, 'null.check', false); $nullable = []; if (is_bool($nullCheck)) { foreach ($input as $name => $type) { $nullable[$name] = $nullCheck; } } else { if (is_array($nullCheck)) { foreach ($input as $name => $type) { $nullable[$name] = false; } foreach ($nullCheck as $index => $n) { grokit_assert(is_string($n), 'BloomFilster null.check has invalid value at position ' . $index); grokit_assert(array_key_exists($n, $nullable), 'BloomFilster null.check has unknown input ' . $n . ' at position ' . $index); $nullable[$n] = true; } } else { grokit_error('BloomFilster null.check must be boolean or list of inputs to check for nulls'); } } $debug = get_default($t_args, 'debug', 0); $bits = pow(2, $exp); $bytes = ceil($bits / 8.0); // Calculate the number of bits set for every possible value of a byte $nBits = []; for ($i = 0; $i < 256; $i++) { $n = $i; $b = 0; while ($n > 0) { $n &= $n - 1; $b++; } $nBits[$i] = $b; } $className = generate_name('BloomFilter'); ?> class <?php echo $className; ?> { static constexpr size_t BITS = <?php echo $bits; ?> ; static constexpr size_t BYTES = <?php echo $bytes; ?> ; static constexpr size_t MASK = BITS - 1; static constexpr std::array<unsigned char, 256> BITS_SET = { <?php echo implode(', ', $nBits); ?> }; static constexpr std::array<unsigned char, 8> BIT_MASKS = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; size_t count; std::array<unsigned char, BYTES> set; //unsigned char set[BYTES]; //std::bitset<BITS> set; public: <?php echo $className; ?> () : count(0), set() { for( size_t i = 0; i < BYTES; i++ ) { //> set[i] = 0; } } ~<?php echo $className; ?> () { } void AddItem( <?php echo const_typed_ref_args($input); ?> ) { count++; <?php foreach ($nullable as $name => $check) { if ($check) { ?> if( IsNull( <?php echo $name; ?> ) ) return; <?php } // if checking for nulls } // foreach input ?> size_t hashVal = H_b; <?php foreach ($input as $name => $type) { ?> hashVal = CongruentHash(Hash(<?php echo $name; ?> ), hashVal); <?php } // foreach input ?> hashVal = hashVal & MASK; const size_t bucket = hashVal >> 3; const size_t bucket_index = hashVal & 0x07; const unsigned char mask = BIT_MASKS[bucket_index]; set[bucket] |= mask; } void AddState( <?php echo $className; ?> & o ) { count += o.count; for( size_t i = 0; i < BYTES; i++ ) { //> set[i] |= o.set[i]; } } void GetResult( <?php echo $outputType; ?> & <?php echo $outputName; ?> ) { size_t nBitsSet = 0; constexpr long double bits = static_cast<long double>(BITS); for( size_t i = 0; i < BYTES; i++ ) { //> nBitsSet += BITS_SET[set[i]]; } long double bitsSet = static_cast<long double>(nBitsSet); if( nBitsSet == BITS ) { // All Bits set, just give the cardinality as an estimate. <?php echo $outputName; ?> = count; } else { long double cardinality = - bits * std::log(1 - (bitsSet / bits)); <?php echo $outputName; ?> = cardinality; } <?php if ($debug > 0) { ?> std::cout << "BloomFilter:" << " bitsSet(" << bitsSet << ")" << " bits(" << bits << ")" << " cardinality(" << cardinality << ")" << " output(" << <?php echo $outputName; ?> << ")" << std::endl;; //> <?php } // if debugging enabled ?> } }; // Storage for static members constexpr std::array<unsigned char, 256> <?php echo $className; ?> ::BITS_SET; constexpr std::array<unsigned char, 8> <?php echo $className; ?> ::BIT_MASKS; <?php $system_headers = ['cmath', 'array']; if ($debug > 0) { $system_headers[] = 'iostream'; } return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'user_headers' => ['HashFunctions.h'], 'system_headers' => $system_headers]; }
function ExtremeTuples(array $t_args, array $inputs, array $outputs) { $extremes = get_first_key($t_args, ['extremes']); $nExt = \count($extremes); grokit_assert($nExt > 0, 'No extremes specified for ExtremeTuples GLA.'); if (\count($inputs) == 0) { grokit_assert(array_key_exists('inputs', $t_args), 'No arguments specified for ExtremeTuples GLA.'); $count = 0; foreach ($t_args['inputs'] as $type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Only datatypes can be specified as inputs to ' . 'the ExtremeTuples GLA'); $name = 'et_val' . $count; $inputs[$name] = $type; } } $outputMap = []; reset($outputs); foreach ($inputs as $name => $type) { $oKey = key($outputs); $outputs[$oKey] = $type; $outputMap[$oKey] = $name; next($outputs); } grokit_assert($nExt <= \count($inputs), 'There can not be more extreme values than there are inputs!'); $mainAtts = []; $extraAtts = []; $minOpts = ['MIN', 'MINIMUM', '-', '<']; $maxOpts = ['MAX', 'MAXIMUM', '+', '>']; $inArrayCase = function ($needle, $haystack) { foreach ($haystack as $item) { if (strcasecmp($needle, $item) == 0) { return true; } } return false; }; $minimum = []; foreach ($extremes as $name => $val) { grokit_assert(array_key_exists($name, $inputs), "ExtremeTuples: Expression with name " . $name . " specified as extreme not found in inputs"); } foreach ($inputs as $name => $type) { if (array_key_exists($name, $extremes)) { $mainAtts[$name] = $type; if ($inArrayCase($extremes[$name], $minOpts)) { $minimum[$name] = true; } else { if ($inArrayCase($extremes[$name], $maxOpts)) { $minimum[$name] = false; } else { grokit_error('Unknown extreme type ' . $extremes[$name] . ' specified for ' . $name); } } } else { $extraAtts[$name] = $type; } } $debug = get_default($t_args, 'debug', 0); $className = generate_name('ExtremeTuples'); ?> class <?php echo $className; ?> { struct Tuple { <?php foreach ($inputs as $name => $type) { ?> <?php echo $type; ?> <?php echo $name; ?> ; <?php } // foreach input ?> // Default Constructor, Copy Constructor, and Copy Assignment are all // default Tuple(void) = default; Tuple(const Tuple &) = default; Tuple & operator = (const Tuple &) = default; Tuple(<?php echo array_template('const {val} & _{key}', ', ', $inputs); ?> ) : <?php echo array_template('{key}(_{key})', ', ', $inputs); ?> { } // operator > means that this tuple is "better" than the other tuple. bool operator > ( const Tuple & other ) const { <?php foreach ($mainAtts as $name => $type) { $op1 = $minimum[$name] ? '<' : '>'; $op2 = !$minimum[$name] ? '<' : '>'; ?> if( <?php echo $name; ?> <?php echo $op1; ?> other.<?php echo $name; ?> ) return true; else if( <?php echo $name; ?> <?php echo $op2; ?> other.<?php echo $name; ?> ) return false; <?php } // foreach main attribute ?> return false; } bool operator < ( const Tuple& other ) const { return other > *this; } bool operator <= (const Tuple & other ) const { return ! (*this > other ); } bool operator >= (const Tuple & other ) const { return !( other > *this ); } bool operator == (const Tuple & other ) const { bool ret = true; <?php foreach ($mainAtts as $name => $type) { ?> ret &= <?php echo $name; ?> == other.<?php echo $name; ?> ; <?php } // foreach main attribute ?> return ret; } }; // struct Tuple typedef std::vector<Tuple> TupleVector; public: class Iterator { public: typedef TupleVector::const_iterator iter_type; private: iter_type begin; iter_type end; public: Iterator(void) = default; Iterator(const Iterator &) = default; Iterator( const iter_type & _begin, const iter_type & _end ) : begin(_begin), end(_end) { } Iterator( const iter_type && _begin, const iter_type && _end ) : begin(_begin), end(_end) { } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { if( begin != end ) { <?php foreach ($outputs as $name => $type) { ?> <?php echo $name; ?> = begin-><?php echo $outputMap[$name]; ?> ; <?php } ?> begin++; return true; } else { return false; } } }; private: uintmax_t __count; // number of tuples covered TupleVector tuples; // Iterator for multi output type Iterator multiIterator; public: // Constructor and destructor <?php echo $className; ?> (void) : __count(0), tuples(), multiIterator() { } ~<?php echo $className; ?> () { } void AddItem( <?php echo const_typed_ref_args($inputs); ?> ) { ++__count; Tuple t(<?php echo args($inputs); ?> ); if( tuples.empty() ) { tuples.push_back(t); } else if( t > tuples.front() ) { tuples.clear(); tuples.push_back(t); } else if( t == tuples.front() ) { tuples.push_back(t); } } void AddState( <?php echo $className; ?> & other ) { if( tuples.size() == 0 ) { tuples.swap(other.tuples); } else if( other.tuples.size() == 0 ) { // Do nothing } else if( tuples.front() > other.tuples.front() ) { // fast path } else if( other.tuples.front() > tuples.front() ) { tuples.swap(other.tuples); } else { for( Tuple & t : other.tuples ) { tuples.push_back(t); } } } void Finalize( void ) { multiIterator = Iterator(tuples.cbegin(), tuples.cend()); } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { return multiIterator.GetNextResult(<?php echo args($outputs); ?> ); } }; // class <?php echo $className; ?> <?php $system_headers = ['vector', 'algorithm', 'cinttypes']; if ($debug > 0) { $system_headers = array_merge($system_headers, ['iostream', 'sstream', 'string']); } return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => 'multi', 'system_headers' => $system_headers); }
<?php $functions[] = ['IsNull', ['@type'], 'BASE::BOOL', true, true]; ?> inline bool IsNull( @type f ) { return isnan(f); } <?php $gContent .= ob_get_clean(); ?> ////////////// // Operators // all operators defined by C++ <?php return array('kind' => 'TYPE', "system_headers" => array("cstdlib", "cstdio", "cinttypes", 'cmath'), "complex" => false, "global_content" => $gContent, 'binary_operators' => ['+', '-', '*', '/', '==', '!=', '>', '<', '>=', '<='], 'unary_operators' => ['+', '-'], 'constructors' => $constructors, 'functions' => $functions, 'properties' => ['real', 'numeric', '_primative_'], 'describe_json' => DescribeJson('float'), 'extras' => ['size.bytes' => 8]); } //end of function // Define the constructors from the various other types. foreach (['BYTE', 'SMALLINT', 'BIGINT', 'FLOAT', 'INT'] as $type) { $fullType = 'base::' . $type; $call = function ($args, $targs = []) use($fullType) { $arg = lookupType($fullType); $ret = lookupType('base::DOUBLE'); return array('kind' => 'FUNCTION', 'input' => [$arg], 'result' => $ret, 'deterministic' => true); }; declareFunctionGlobal('base', 'DOUBLE', [$fullType], $call); }
/** * A GLA that estimates the cardinality of a dataset using the HyperLogLog * algorithm, with a configurable number of bins. */ function HyperLogLog(array $t_args, array $input, array $output) { $debug = get_default($t_args, 'debug', 0); grokit_assert(\count($output) == 1, 'HyperLogLog produces only 1 value, ' . \count($output) . ' outputs given.'); $outputName = array_keys($output)[0]; $outputType = array_get_index($output, 0); if (is_null($outputType)) { $outputType = lookupType('BASE::BIGINT'); } $output[$outputName] = $outputType; grokit_assert($outputType->is('numeric'), 'BloomFilter output must be numeric!'); $exp = get_first_key_default($t_args, ['bins.exponent'], 4); grokit_assert(is_integer($exp), 'HyperLogLog bins.exponent must be an integer'); // Set limit of 2^24 bins, because states past 16MB start to get silly grokit_assert($exp >= 4 && $exp < 24, 'HyperLogLog bins.exponent must be in range [4, 24]'); $useBuiltinCtz = get_default($t_args, 'use.builtin.ctz', true); $ctzFunc = $useBuiltinCtz ? '__builtin_ctzl' : 'ctz'; $bins = pow(2, $exp); // Determine the value of alpha based on $exp switch ($exp) { case 4: $alpha = 0.673; break; case 5: $alpha = 0.697; break; case 6: $alpha = 0.709; break; default: $alpha = 0.7213000000000001 / (1 + 1.079 / $bins); } $className = generate_name('HyperLogLog'); ?> class <?php echo $className; ?> { // Number of bins for registers static constexpr const size_t NUM_BINS = <?php echo $bins; ?> ; // Number of bits used to index into registers, log2(NUM_BINS) static constexpr const size_t INDEX_BITS = <?php echo $exp; ?> ; // Mask used to obtain register index from hash value static constexpr const size_t INDEX_MASK = NUM_BINS - 1; // Alpha coefficient used to correct cardinality estimate. Based on NUM_BINS. static constexpr const long double ALPHA = <?php echo $alpha; ?> ; // Value of cardinality estimate after which we must apply the // large range correction static constexpr const long double LARGE_BREAKPOINT = (1.0 / 30.0) * <?php echo pow(2, 32); ?> ; // Constants for population count static constexpr const uint64_t m1 = 0x5555555555555555; static constexpr const uint64_t m2 = 0x3333333333333333; static constexpr const uint64_t m4 = 0x0f0f0f0f0f0f0f0f; static constexpr const uint64_t h01 = 0x0101010101010101; // The registers std::array<unsigned char, NUM_BINS> registers; // A count used to remember how many tuples were processed, mostly for debugging. size_t count; public: <?php echo $className; ?> (void) : registers() { for( auto & elem : registers ) { elem = 0; } } ~<?php echo $className; ?> () { } int popcount(uint64_t x) { // Put count of each 2 bits into those 2 bits x -= (x >> 1) & m1; // Put count of each 4 bits into those 4 bits x = (x & m2) + ((x >> 2) & m2); // Put count of each 8 bits into those 8 bits x = (x + (x >> 4)) & m4; // Returns left 8 bits of x + (x << 8) + (x << 16) + ... return (x * h01) >> 56; } int ctz(int64_t x) { return popcount((x & -x) - 1); } void AddItem( <?php echo const_typed_ref_args($input); ?> ) { count++; uint64_t hashVal = H_b; <?php foreach ($input as $name => $type) { ?> hashVal = CongruentHash(Hash(<?php echo $name; ?> ), hashVal); <?php } // for each input ?> const size_t registerIndex = hashVal & INDEX_MASK; uint64_t value = hashVal >> INDEX_BITS; unsigned char nZeros = <?php echo $ctzFunc; ?> (value); unsigned char & registerValue = registers[registerIndex]; registerValue = registerValue > nZeros ? registerValue : nZeros; } void AddState( <?php echo $className; ?> & other ) { for( size_t i = 0; NUM_BINS > i; i++ ) { unsigned char & rVal = registers[i]; unsigned char & oVal = other.registers[i]; rVal = rVal > oVal ? rVal : oVal; } } void GetResult( <?php echo $outputType; ?> & <?php echo $outputName; ?> ) { // Compute harmonic sum of registers and correct by alpha long double cardEst = 0; size_t nZeroRegisters = 0; for( auto elem : registers ) { long double power = - static_cast<long double>(elem); cardEst += std::pow(2.0, power); if( elem == 0 ) nZeroRegisters++; } const long double nBins = static_cast<long double>(NUM_BINS); const long double zeroBins = static_cast<long double>(nZeroRegisters); cardEst = 1 / cardEst; cardEst *= ALPHA * nBins * nBins; long double cardinality = cardEst; if( (cardEst < 2.5 * NUM_BINS) ) { //> // Possible small range correction if( nZeroRegisters > 0 ) { // Small range correction cardinality = nBins * std::log(nBins / zeroBins); } } // TODO: Figure out if the large range correction is needed for 64-bit // hashes. <?php echo $outputName; ?> = cardinality; } }; <?php $system_headers = ['cmath', 'array', 'cinttypes']; if ($debug > 0) { $system_headers[] = 'iostream'; } return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'user_headers' => ['HashFunctions.h'], 'system_headers' => $system_headers]; }
inline bool IsNull( @type f ) { return isnan(f); } <?php $gContents = ob_get_clean(); ?> ////////////// // Operators // all operators defined by C++ <?php return array('kind' => 'TYPE', "system_headers" => array("cstdlib", "cstdio", "cinttypes", 'cmath', 'limits'), "complex" => false, 'binary_operators' => ['+', '-', '*', '/', '==', '!=', '>', '<', '>=', '<='], 'unary_operators' => ['+', '-'], 'constructors' => $constructors, 'functions' => $functions, 'properties' => ['real', 'numeric', '_primative_'], 'global_content' => $gContents, 'describe_json' => DescribeJson('float'), 'extras' => ['size.bytes' => 4]); } // end of function // Define the constructors from the various other types. foreach (['BYTE', 'SMALLINT', 'BIGINT', 'INT', 'FLOAT', 'DOUBLE'] as $type) { $fullType = 'base::' . $type; $call = function ($args, $targs = []) use($fullType) { $arg = lookupType($fullType); $ret = lookupType('base::FLOAT'); return array('kind' => 'FUNCTION', 'input' => [$arg], 'result' => $ret, 'deterministic' => true); }; declareFunctionGlobal('base', 'FLOAT', [$fullType], $call); } ?>
public static function AlphaOperator($op, $nArgs, DataType $alpha) { $aVal = $alpha->value(); switch ($nArgs) { case 1: grokit_assert(array_key_exists($op, self::$unaryOperators), 'Attempted to generate invalid unary operator ' . $op . ' with alpha type ' . $alpha); $form = self::$unaryOperators[$op]; switch ($form) { case 'ata': $args = [$alpha->value()]; $call = function () use($alpha) { return array('kind' => InfoKind::T_OP, 'input' => [$alpha], 'result' => $alpha); }; break; } break; case 2: // Binary operator grokit_assert(array_key_exists($op, self::$binaryOperators), 'Attempted to generate invalid binary operator ' . $op . ' with alpha type ' . $alpha); $form = self::$binaryOperators[$op]; switch ($form) { case 'aata': $args = [$aVal, $aVal]; $call = function () use($alpha) { return array('kind' => InfoKind::T_OP, 'input' => [$alpha, $alpha], 'result' => $alpha); }; break; case 'aatb': $args = [$aVal, $aVal]; $call = function () use($alpha) { return array('kind' => InfoKind::T_OP, 'input' => [$alpha, $alpha], 'result' => lookupType('base::bool')); }; break; } } declareOperator($op, $args, $call); }
$name = generate_name('Time'); ?> inline TIME <?php echo $name; ?> (const DATETIME& date_time) { return TIME(date_time.Hour(), date_time.Minute(), date_time.Second(), 0); } <?php return ['kind' => 'FUNCTION', 'name' => $name, 'input' => $args, 'result' => $result, 'deterministic' => true]; }); ?> <?php declareFunction('DATETIMEFROMINT', ['BASE::INT'], function ($args) { $result = lookupType('BASE::DATETIME'); $name = generate_name('DateTimeFromInt'); ?> inline DATETIME <?php echo $name; ?> (const INT& seconds) { return DATETIME(seconds); } <?php return ['kind' => 'FUNCTION', 'name' => $name, 'input' => $args, 'result' => $result, 'deterministic' => true]; });
function page_rank_vd($t_args, $inputs, $outputs) { $className = generate_name('PageRank'); $inputs_ = array_combine(['s', 't'], $inputs); $vertex = $inputs_['s']; $outputs_ = ['node' => $vertex, 'rank' => lookupType('double')]; $outputs = array_combine(array_keys($outputs), $outputs_); $sys_headers = ['vector', 'algorithm']; $user_headers = []; $lib_headers = []; $libraries = []; $properties = []; $extra = []; $result_type = ['fragment']; ?> using namespace std; class <?php echo $className; ?> ; <?php $constantState = lookupResource('pagerankVD::page_rank_vd_Constant_State', ['className' => $className]); ?> class <?php echo $className; ?> { public: using ConstantState = <?php echo $constantState; ?> ; using Iterator = std::pair<int, int>; static const constexpr float kDamping = 0.85; static const constexpr int nIterations = 5; static const constexpr int kBlock = 32; static const constexpr int kMaxFragments = 64; private: static std::vector<double> weight; static std::vector<double> pagerank; static std::vector<double> oldPagerank; const ConstantState& constant_state; long nVertices; int iteration; int num_fragments; public: <?php echo $className; ?> (const <?php echo $constantState; ?> & state) : constant_state(state), nVertices(state.nVertices), iteration(state.iteration) { } void AddItem(<?php echo const_typed_ref_args($inputs_); ?> ) { if (iteration == 0) { nVertices = max((long) max(s, t), nVertices); return; } else if (iteration == 1) { weight[s]++; } else { oldPagerank[t] += weight[s] * pagerank[s]; } } void AddState(<?php echo $className; ?> &other) { if (iteration == 0) nVertices = max(nVertices, other.nVertices); } // Most computation that happens at the end of each iteration is parallelized // by performed it inside Finalize. bool ShouldIterate(ConstantState& state) { state.iteration = ++iteration; cout << "Finished Iteration: " << iteration << endl; if (iteration == 1) { state.nVertices = ++nVertices; cout << "num_nodes: " << nVertices << endl; oldPagerank.reserve(nVertices); pagerank.reserve(nVertices); weight.reserve(nVertices); std::fill (pagerank.begin(),pagerank.end(),1); return true; } else { return iteration < nIterations + 1; } } int GetNumFragments() { long size = (nVertices - 1) / kBlock + 1; // nVertices / kBlock rounded up. num_fragments = (iteration == 0) ? 0 : min(size, (long) kMaxFragments); cout << "Returning " << num_fragments << " fragments" << endl; return num_fragments; } Iterator* Finalize(long fragment) { int count = nVertices; int first = fragment * (count / kBlock) / num_fragments * kBlock; int final = (fragment == num_fragments - 1) ? count - 1 : (fragment + 1) * (count / kBlock) / num_fragments * kBlock - 1; if (iteration == 2) { for(int i = first; i <= final; i++){ pagerank[i] = 1; weight[i] = 1/weight[i]; oldPagerank[i] = 0; } } else { for(int i = first; i <= final; i++){ pagerank[i] = (1 - kDamping) + kDamping * oldPagerank[i]; oldPagerank[i] = 0; } } return new Iterator(first, final); } bool GetNextResult(Iterator* it, <?php echo typed_ref_args($outputs_); ?> ) { if (iteration < nIterations + 1) return false; if (it->first > it->second) return false; node = it->first; rank = pagerank[it->first]; it->first++; return true; } }; // Initialize the static member types. vector<double> <?php echo $className; ?> ::weight; vector<double> <?php echo $className; ?> ::pagerank; vector<double> <?php echo $className; ?> ::oldPagerank; typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'properties' => $properties, 'extra' => $extra, 'iterable' => true, 'intermediates' => true, 'input' => $inputs, 'output' => $outputs, 'result_type' => $result_type, 'generated_state' => $constantState]; }
/** * A GLA that counts the number of distinct elements by keeping track of the * distinct elements. * * Unless an exact count of the distinct is absolutely needed, consider using * an approximation of the distinct, such as a Bloom Filter. */ function CountDistinct(array $t_args, array $input, array $output) { grokit_assert(\count($output) == 1, 'CountDistinct should have only 1 output, ' . \count($output) . 'given'); $outputName = array_keys($output)[0]; $outputType = array_get_index($output, 0); if (is_null($outputType)) { $outputType = lookupType('BASE::BIGINT'); } $output[$outputName] = $outputType; grokit_assert($outputType->is('numeric'), 'CountDistinct output must be numeric!'); $useMCT = get_default($t_args, 'use.mct', true); $keepHashes = get_default($t_args, 'mct.keep.hashes', false); $initSize = get_default($t_args, 'init.size', 65536); $nullCheck = get_default($t_args, 'null.check', false); grokit_assert(is_bool($useMCT), 'CountDistinct use.mct argument must be boolean'); grokit_assert(is_integer($initSize), 'Distinct init.size argument must be an integer'); grokit_assert($initSize > 0, 'Distinct init.size argument must be positive'); grokit_assert(is_bool($keepHashes), 'CountDistinct mct.keep.hashes argument must be boolean'); $distTmpArgs = ['use.mct' => $useMCT, 'init.size' => $initSize, 'mct.keep.hashes' => $keepHashes, 'null.check' => $nullCheck]; $gla = lookupGLA('BASE::DISTINCT', $distTmpArgs, $input, $input); $className = generate_name('CountDistinct'); ?> class <?php echo $className; ?> { using Distinct = <?php echo $gla->value(); ?> ; Distinct distinctGLA; public: <?php echo $className; ?> (void): distinctGLA() { } ~<?php echo $className; ?> (void) { } void AddItem(<?php echo const_typed_ref_args($input); ?> ) { distinctGLA.AddItem(<?php echo args($input); ?> ); } void AddState(<?php echo $className; ?> & o) { distinctGLA.AddState(o.distinctGLA); } void GetResult(<?php echo $outputType; ?> & <?php echo $outputName; ?> ) { <?php echo $outputName; ?> = distinctGLA.get_countDistinct(); } }; <?php return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single']; }
<?php // Copyright 2015, Tera Insights, LLC. All Rights Reserved. declareFunctionGlobal('BASE', 'BOOL', ['BASE::NULL'], function () { $boolType = lookupType('BASE::BOOL'); $nullType = lookupType('BASE::NULL'); return ['kind' => 'FUNCTION', 'name' => 'bool_Null', 'input' => [$nullType], 'result' => $boolType, 'deterministic' => true]; }); ?> inline int ToString(bool b, char * buffer) { if (b) return 1 + sprintf(buffer, "true"); else return 1 + sprintf(buffer, "false"); } inline void FromString(bool & b, const char* buffer) { char first = *buffer; char upper = *buffer & 0xDF; if (first == '1' || upper == 'T' || upper == 'Y') b = true; else b = false; } namespace BASE { inline bool bool_Null(const GrokitNull& null) { return false; }
inline int64_t ClusterValue(const @type & x) { return x; } <?php $gContent .= ob_get_clean(); ?> ////////////// // Operators // all operators defined by C++ <?php return array('kind' => 'TYPE', "system_headers" => array("cstdlib", "cstdio", "cinttypes", 'limits'), "complex" => false, "global_content" => $gContent, 'binary_operators' => ['+', '-', '*', '/', '%', '==', '!=', '>', '<', '>=', '<='], 'unary_operators' => ['+', '-'], 'constructors' => $constructors, 'functions' => $functions, 'properties' => ['integral', 'numeric', '_primative_', 'clusterable'], 'describe_json' => DescribeJson('integer'), 'extras' => ['size.bytes' => 4]); } declareOperator('//', ['base::UINT', 'base::UINT'], function ($args) { $l = lookupType('base::UINT'); return ['kind' => 'OPERATOR', 'name' => '/', 'input' => [$l, $l], 'result' => $l, 'deterministic' => true]; }); // Define the constructors from the various other types. foreach (['BYTE', 'SMALLINT', 'INT', 'UINT', 'BIGINT', 'FLOAT', 'DOUBLE'] as $type) { $fullType = 'base::' . $type; $call = function ($args, $targs = []) use($fullType) { $arg = lookupType($fullType); $ret = lookupType('base::UINT'); return array('kind' => 'FUNCTION', 'input' => [$arg], 'result' => $ret, 'deterministic' => true); }; declareFunctionGlobal('base', 'UINT', [$fullType], $call); }
function parseNull($ast) { assert_ast_type($ast, Nodetype::NUL); $nType = lookupType('BASE::NULL'); $nVal = 'GrokitNull::Value'; $source = ast_node_source($ast); $info = new ExpressionInfo($source, $nType, $nVal, true); return $info; }
public function apply($obj, array $exprs, $source) { $this->checkFuzzy($this->args, $exprs, 'parameter'); $exprVals = array_map(function ($val) { return $val->value(); }, $exprs); $value = $obj->value() . '.' . $this->name . '(' . implode(', ', $exprVals) . ')'; $is_const = $this->deterministic && $obj->is_const(); foreach ($exprs as $expr) { $is_const = $is_const && $expr->is_const(); } $retType = lookupType($this->resultType); $info = new ExpressionInfo($source, $retType, $value, $is_const); $info->absorbMeta($obj); foreach ($exprs as $expr) { $info->absorbMeta($expr); } return $info; }
function CCompLP($t_args, $inputs, $outputs) { // Class name is randomly generated. $className = generate_name('CComp'); // Initializiation of argument names. $inputs_ = array_combine(['s', 't'], $inputs); $vertex = $inputs_['s']; // Construction of outputs. $outputs_ = ['node' => $vertex, 'component' => lookupType('int')]; $outputs = array_combine(array_keys($outputs), $outputs_); $sys_headers = ['armadillo', 'algorithm']; $user_headers = []; $lib_headers = []; $libraries = ['armadillo']; $properties = []; $extra = []; $result_type = ['multi']; ?> using namespace arma; using namespace std; class <?php echo $className; ?> ; <?php $constantState = lookupResource('graph::CCompLP_Constant_State', ['className' => $className]); ?> class <?php echo $className; ?> { public: // The constant state for this GLA. using ConstantState = <?php echo $constantState; ?> ; // The number of iterations to perform, not counting the initial set-up. static const constexpr int kIterations = 30; // The work is split into chunks of this size before being partitioned. static const constexpr int kBlock = 32; // The maximum number of fragments to use. static const constexpr int kMaxFragments = 64; private: // Node Component static arma::rowvec node_component; // The typical constant state for an iterable GLA. const ConstantState& constant_state; // The number of unique nodes seen. long num_nodes; // long output_iterator; // The current iteration. int iteration; // check if need more iterations long connections; public: <?php echo $className; ?> (const <?php echo $constantState; ?> & state) : constant_state(state), num_nodes(state.num_nodes), iteration(state.iteration),output_iterator(0),connections(0) { } // Basic dynamic array allocation. void AddItem(<?php echo const_typed_ref_args($inputs_); ?> ) { if (iteration == 0) { num_nodes = max((long) max(s, t), num_nodes); return; } /*else if (iteration == 1){ long max_known = (long) max(s, t); node_component(s) = max_known; node_component(t) = max_known; ++ connections; }*/else{ long s_id = node_component(s), t_id = node_component(t); if (s_id != t_id){ ++ connections; if (s_id > t_id) node_component(t) = s_id; else node_component(s) = t_id; } } } // Hashes are merged. void AddState(<?php echo $className; ?> &other) { if (iteration == 0) num_nodes = max(num_nodes, other.num_nodes); else connections += other.connections; } // Most computation that happens at the end of each iteration is parallelized // by performed it inside Finalize. bool ShouldIterate(ConstantState& state) { state.iteration = ++iteration; if (iteration == 1) {// allocate memory // num_nodes is incremented because IDs are 0-based. state.num_nodes = ++num_nodes; // Allocating space can't be parallelized. node_component.set_size(num_nodes); for (long i = 0; i < num_nodes; ++ i) node_component(i) = i; return true; } else { return connections > 0 && iteration < kIterations + 1; } } // Finalize does nothing void Finalize() {} bool GetNextResult(<?php echo typed_ref_args($outputs_); ?> ) { // should iterate is true if (connections > 0 && iteration < kIterations + 1) return false; if(output_iterator < num_nodes){ node = output_iterator++; component = node_component(node); return true; }else{ return false; } } }; // Initialize the static member types. arma::rowvec <?php echo $className; ?> ::node_component; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'properties' => $properties, 'extra' => $extra, 'iterable' => true, 'input' => $inputs, 'output' => $outputs, 'result_type' => $result_type, 'generated_state' => $constantState]; }
function CATEGORY(array $t_args) { if (array_key_exists('dict', $t_args)) { $values = $t_args['dict']; $maxID = 0; foreach ($values as $id => $val) { $maxID = \max($id, $maxID); } } else { $old_vals = get_first_key($t_args, ['values', 0]); $startAt = get_first_key_default($t_args, ['start.at'], 0); $values = []; $maxID = $startAt; foreach ($old_vals as $ind => $val) { $values[$maxID++] = $val; } } $cardinality = \count($values); // Add 1 to the cardinality for the invalid id $storageTypeBits = ceil(log($maxID + 1, 2)); if ($storageTypeBits > 64) { // This should never happen. PHP would explode processing 2^64 values. grokit_error("Unable to store {$cardinality} values within 64 bits."); } else { if ($storageTypeBits > 32) { $storageType = 'uint64_t'; $storageBytes = 8; } else { if ($storageTypeBits > 16) { $storageType = 'uint32_t'; $storageBytes = 4; } else { if ($storageTypeBits > 8) { $storageType = 'uint16_t'; $storageBytes = 2; } else { $storageType = 'uint8_t'; $storageBytes = 1; } } } } $className = generate_name('CATEGORY'); $stringType = lookupType('base::STRING'); $methods = []; $constructors = []; $functions = []; ?> class <?php echo $className; ?> { public: typedef <?php echo $storageType; ?> StorageType; typedef std::unordered_map<StorageType, std::string> IDToNameMap; typedef std::unordered_map<std::string, StorageType> NameToIDMap; static const StorageType InvalidID __attribute__((weak)); private: static const IDToNameMap idToName __attribute__((weak)); static const NameToIDMap nameToID __attribute__((weak)); // The ID of this categorical variable StorageType myID; public: /* ----- Constructors / Destructor ----- */ <?php echo $className; ?> ( void ); <?php $constructors[] = [['base::STRING_LITERAL'], true]; ?> <?php echo $className; ?> ( const char * ); <?php $constructors[] = [['base::STRING'], true]; ?> <?php echo $className; ?> ( const <?php echo $stringType; ?> & ); <?php echo $className; ?> ( const <?php echo $storageType; ?> ); <?php echo $className; ?> ( const <?php echo $className; ?> & ); <?php $constructors[] = [['BASE::NULL'], true]; ?> <?php echo $className; ?> ( const GrokitNull & ); <?php echo $className; ?> & operator =( const <?php echo $className; ?> & ) = default; ~<?php echo $className; ?> (void) {} /* ----- Methods ----- */ void FromString( const char * ); <?php $methods[] = ['ToString', [], 'base::STRING_LITERAL', true]; ?> const char * ToString( void ) const; StorageType GetID( void ) const; void SetID( StorageType id ); // Determines whether or not the category is valid. <?php $methods[] = ['Invalid', [], 'base::bool', true]; ?> bool Invalid(void) const; <?php $methods[] = ['Valid', [], 'base::bool', true]; ?> bool Valid(void) const; /* ----- Operators ----- */ bool operator ==( const <?php echo $className; ?> & ) const; bool operator !=( const <?php echo $className; ?> & ) const; bool operator <( const <?php echo $className; ?> & ) const; bool operator <=( const <?php echo $className; ?> & ) const; bool operator >( const <?php echo $className; ?> & ) const; bool operator >=( const <?php echo $className; ?> & ) const; // Implicit conversion to storage type operator <?php echo $storageType; ?> () const; // To/From Json void toJson( Json::Value & dest ) const; void fromJson( const Json::Value & src ); }; /* ----- Constructors ----- */ inline <?php echo $className; ?> :: <?php echo $className; ?> ( void ) : myID(InvalidID) { } inline <?php echo $className; ?> :: <?php echo $className; ?> ( const char * str ) { FromString(str); } inline <?php echo $className; ?> :: <?php echo $className; ?> ( const <?php echo $stringType; ?> & str ) { FromString(str.ToString()); } inline <?php echo $className; ?> :: <?php echo $className; ?> ( const <?php echo $storageType; ?> val ) : myID(val) { } inline <?php echo $className; ?> :: <?php echo $className; ?> ( const <?php echo $className; ?> & other ) : myID(other.myID) { } inline <?php echo $className; ?> :: <?php echo $className; ?> ( const GrokitNull & nullval ) : myID(InvalidID) { } /* ----- Methods ----- */ inline void <?php echo $className; ?> :: FromString( const char * str ) { auto it = nameToID.find(str); if( it != nameToID.end() ) { myID = it->second; } else { myID = InvalidID; } } inline const char * <?php echo $className; ?> :: ToString( void ) const { auto it = idToName.find(myID); if( it != idToName.end() ) { return it->second.c_str(); } else { return "NULL"; } } inline auto <?php echo $className; ?> :: GetID( void ) const -> StorageType { return myID; } inline void <?php echo $className; ?> :: SetID( StorageType id ) { myID = id; } inline bool <?php echo $className; ?> :: Valid(void) const { return idToName.count(myID) > 0; } inline bool <?php echo $className; ?> :: Invalid(void) const { return ! Valid(); } /* ----- Operators ----- */ inline bool <?php echo $className; ?> :: operator ==( const <?php echo $className; ?> & other ) const { return myID == other.myID; } inline bool <?php echo $className; ?> :: operator !=( const <?php echo $className; ?> & other ) const { return myID != other.myID; } inline bool <?php echo $className; ?> :: operator <( const <?php echo $className; ?> & other ) const { return myID < other.myID; } inline bool <?php echo $className; ?> :: operator >( const <?php echo $className; ?> & other ) const { return myID > other.myID; } inline bool <?php echo $className; ?> :: operator <=( const <?php echo $className; ?> & other ) const { return myID <= other.myID; } inline bool <?php echo $className; ?> :: operator >=( const <?php echo $className; ?> & other ) const { return myID >= other.myID; } // To/From Json inline void <?php echo $className; ?> :: toJson( Json::Value & dest ) const { dest = (Json::Int64) myID; } inline void <?php echo $className; ?> :: fromJson( const Json::Value & src ) { myID = (StorageType) src.asInt64(); } inline <?php echo $className; ?> :: operator <?php echo $storageType; ?> () const { return myID; } <?php ob_start(); $functions[] = ['Hash', ['@type'], 'base::BIGINT', true, true]; ?> template<> inline uint64_t Hash(const @type & thing) { return thing.GetID(); } inline void FromString( @type & c, const char * str ) { c.FromString(str); } inline int ToString( const @type & c, char * buffer ) { const char * str = c.ToString(); strcpy( buffer, str); int len = strlen(buffer); return len + 1; } inline void ToJson( const @type & src, Json::Value & dest ) { src.toJson(dest); } inline void FromJson( const Json::Value & src, @type & dest ) { dest.fromJson(src); } <?php $functions[] = ['IsNull', ['@type'], 'BASE::BOOL', true, true]; ?> inline bool IsNull( const @type c ) { return c.Invalid(); } <?php $globalContents = ob_get_clean(); ?> // Initialize static values const <?php echo $className; ?> ::IDToNameMap <?php echo $className; ?> :: idToName = { <?php echo array_template('{{key},"{val}"}', ',', $values); ?> }; const <?php echo $className; ?> ::NameToIDMap <?php echo $className; ?> :: nameToID = { <?php echo array_template('{"{val}",{key}}', ',', $values); ?> }; const <?php echo $className; ?> ::StorageType <?php echo $className; ?> :: InvalidID = std::numeric_limits<<?php echo $className; ?> ::StorageType>::max(); <?php return ['kind' => 'TYPE', 'name' => $className, 'properties' => ['categorical'], 'extras' => ['cardinality' => $cardinality, 'size.bytes' => $storageBytes], 'binary_operators' => ['==', '!=', '<', '>', '<=', '>='], 'system_headers' => ['cinttypes', 'unordered_map', 'string', 'cstring', 'limits'], 'global_content' => $globalContents, 'complex' => false, 'methods' => $methods, 'constructors' => $constructors, 'functions' => $functions, 'describe_json' => DescribeJson('factor', DescribeJsonStatic(['levels' => $values]))]; }
function ConnectedComponents(array $t_args, array $inputs, array $outputs) { // Class name is randomly generated $className = generate_name("CCompGLA"); // Processing of inputs. grokit_assert(count($inputs) == 2, 'Connected Components: 2 inputs expected'); $inputs_ = array_combine(['src', 'dst'], $inputs); // Setting output type $outType = lookupType('int'); $outputs_ = ['node' => $outType, 'component' => $outType]; $outputs = array_combine(array_keys($outputs), $outputs_); $sys_headers = ["vector", "mct/hash-map.hpp"]; $user_headers = []; $lib_headers = []; ?> using namespace std; class <?php echo $className; ?> ; class <?php echo $className; ?> { class UnionFindMap{ private: mct::closed_hash_map<uint64_t, uint64_t>* parent; mct::closed_hash_map<uint64_t, uint64_t> sz; const uint64_t NON_EXISTING_ID = -1; public: // constructor did nothing UnionFindMap(){ parent = new mct::closed_hash_map<uint64_t, uint64_t>(); } uint64_t Find(uint64_t i){ if ((*parent).find(i) == (*parent).end()){ return NON_EXISTING_ID; } // use path compression here while (i != (*parent)[i]){ (*parent)[i] = (*parent)[(*parent)[i]]; i = (*parent)[i]; } return i; } // put merge small tree into higher tree // if disjoint, merge and return false void Union(uint64_t i, uint64_t j){ uint64_t ip = Find(i); uint64_t jp = Find(j); if (ip != NON_EXISTING_ID && jp != NON_EXISTING_ID){// both exists if (ip != jp){ if (sz[ip] < sz[jp]){ (*parent)[ip] = jp; sz[jp] += sz[ip]; }else{ (*parent)[jp] = ip; sz[ip] += sz[jp]; } } }else if(ip == NON_EXISTING_ID && jp == NON_EXISTING_ID){// both new (*parent)[i] = i; sz[i] = 2; (*parent)[j] = i; }else if (jp == NON_EXISTING_ID){ // i exists (*parent)[j] = ip; sz[ip] ++; }else{ (*parent)[i] = jp; sz[jp] ++; } } mct::closed_hash_map<uint64_t, uint64_t>* GetUF(){ return parent; } bool IsEmpty(){ return (*parent).empty(); } uint64_t GetSize(){ return (uint64_t) (*parent).size(); } void SetData(mct::closed_hash_map<uint64_t, uint64_t>* other_data){ parent = other_data; } // void FinalizeRoot(){ for(mct::closed_hash_map<uint64_t, uint64_t>::iterator it = (*parent).begin(); it != (*parent).end(); ++ it){ it->second = Find(it->first); } } void Clear(){ (*parent).clear(); sz.clear(); } ~UnionFindMap(){ delete parent; } }; private: // union-find map data structure, which contains nodeID->compID information UnionFindMap primary_uf; mct::closed_hash_map<uint64_t, uint64_t>::iterator output_iterator, output_iterator_end; bool localFinalized = false; public: <?php echo $className; ?> () {} void AddItem(<?php echo const_typed_ref_args($inputs_); ?> ) { uint64_t src_ = Hash(src); uint64_t dst_ = Hash(dst); primary_uf.Union(src_, dst_); } void AddState(<?php echo $className; ?> &other) { FinalizeLocalState(); other.FinalizeLocalState(); mct::closed_hash_map<uint64_t, uint64_t>* this_state_data = primary_uf.GetUF(); mct::closed_hash_map<uint64_t, uint64_t>* other_state_data = other.primary_uf.GetUF(); if (primary_uf.GetSize() < other.primary_uf.GetSize()){ mct::closed_hash_map<uint64_t, uint64_t>* tmp = this_state_data; this_state_data = other_state_data; other_state_data = tmp; primary_uf.SetData(this_state_data); other.primary_uf.SetData(other_state_data); } assert(primary_uf.GetSize() >= other.primary_uf.GetSize()); UnionFindMap secondary_uf; //go over the other state, and maintain a secondary table for(auto const& entry:(*other_state_data)){ if ((*this_state_data).count(entry.first) == 1){// key exists in this state uint64_t this_comp_id = (*this_state_data)[entry.first]; if (this_comp_id != entry.second) // merge needed secondary_uf.Union(this_comp_id, entry.second); }else{ (*this_state_data)[entry.first] = entry.second; } } // check if side table empty if (secondary_uf.IsEmpty()){ return; } // apply the side table secondary_uf.FinalizeRoot(); mct::closed_hash_map<uint64_t, uint64_t>* secondary_state_data = secondary_uf.GetUF(); for (auto& p:(*this_state_data)){ if ((*secondary_state_data).find(p.second) != (*secondary_state_data).end()){ p.second = (*secondary_state_data)[p.second]; } } } void FinalizeLocalState(){ if (!localFinalized){ primary_uf.FinalizeRoot(); localFinalized = true; } } void Finalize(){ output_iterator = primary_uf.GetUF()->begin(); output_iterator_end = primary_uf.GetUF()->end(); } bool GetNextResult(<?php echo typed_ref_args($outputs_); ?> ) { if (output_iterator != output_iterator_end){ node = output_iterator->first; component = output_iterator->second; ++ output_iterator; return true; }else{ return false; } } }; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'input' => $inputs, 'output' => $outputs, 'result_type' => 'multi']; }
function Multiplexer(array $t_args, array $inputs, array $outputs) { $className = generate_name('Multiplexer'); if (\count($inputs) == 0) { grokit_assert(array_key_exists('input', $t_args), 'No inputs specified for Multiplexer'); $inputs = $t_args['input']; foreach ($t_args['inputs'] as $name => &$type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Only types may be specified as inputs to Multiplexer.'); } $inputs = ensure_valid_names($inputs, 'multi_input'); } $glas = get_first_key($t_args, ['glas', 0]); grokit_assert(\count($glas) > 0, 'No GLAs specified for Multiplexer.'); $myGLAs = []; $glaInputs = []; $glaOutputs = []; $resultType = 'multi'; $usedOutputs = []; $libraries = []; $glaGenStates = []; $glaReqStates = []; $configurable = false; $constArgs = []; $genStates = []; $reqStates = []; $iterable = null; foreach ($glas as $name => $glaInfo) { grokit_assert(is_array($glaInfo), 'Template argument \'glas\' must be an array'); grokit_assert(array_key_exists('gla', $glaInfo), 'No GLA given for glas[' . $name . ']'); grokit_assert(array_key_exists('inputs', $glaInfo), 'No inputs given for glas[' . $name . ']'); grokit_assert(array_key_exists('outputs', $glaInfo), 'No outputs given for glas[' . $name . ']'); $gla = $glaInfo['gla']; $glaInAtts = $glaInfo['inputs']; $glaOutAtts = $glaInfo['outputs']; grokit_assert(is_gla($gla), 'Non-GLA given for glas[' . $name . '][gla]'); grokit_assert(is_array($glaInAtts), 'Non-array given for inputs for gla ' . $name); grokit_assert(is_array($glaOutAtts), 'Non-array given for outputs for gla ' . $name); $glaInAtts = array_map('strval', $glaInAtts); $glaOutAtts = array_map('strval', $glaOutAtts); $glaName = "innerGLA_" . $name; $glaInputs[$glaName] = []; $glaOutputs[$glaName] = []; foreach ($glaInAtts as $att) { grokit_assert(array_key_exists($att, $inputs), 'Input ' . $att . ' for GLA ' . $name . ' not found in inputs'); $glaInputs[$glaName][$att] = $inputs[$att]; } foreach ($glaOutAtts as $att) { grokit_assert(array_key_exists($att, $outputs), 'Output ' . $att . ' for GLA ' . $name . ' not found in outputs'); grokit_assert(!in_array($att, $usedOutputs), 'Output ' . $att . ' used by multiple GLAs'); $usedOutputs[] = $att; $glaOutputs[$glaName][$att] = $outputs[$att]; } //fwrite(STDERR, "Inputs for GLA " . $glaName . ": " . print_r($glaInputs[$glaName], true) . PHP_EOL ); //fwrite(STDERR, "Outputs for GLA " . $glaName . ": " . print_r($glaOutputs[$glaName], true) . PHP_EOL ); $gla = $gla->apply($glaInputs[$glaName], $glaOutputs[$glaName]); $myGLAs[$glaName] = $gla; $glaRez[$glaName] = get_first_value($gla->result_type(), ['multi', 'single', 'state']); $libraries = array_merge($libraries, $gla->libraries()); if ($glaRez[$glaName] == 'state') { grokit_assert(\count($glaOutputs[$glaName]) == 1, "GLA {$glaName} is produced as state, and thus must have exactly 1 output."); $stateType = lookupType('base::STATE', ['type' => $gla]); $glaOutputs[$glaName] = array_combine(array_keys($glaOutputs[$glaName]), [$stateType]); } else { grokit_assert(\count($glaOutputs[$glaName]) == \count($gla->output()), 'GLA ' . $glaName . ' produces different number of outputs than expected'); $glaOutputs[$glaName] = array_combine(array_keys($glaOutputs[$glaName]), $gla->output()); } // Set types for our output foreach ($glaOutputs[$glaName] as $attName => $type) { $outputs[$attName] = $type; } if (is_null($iterable)) { $iterable = $gla->iterable(); } else { grokit_assert($iterable == $gla->iterable(), 'Multiplexer does not support mixing iterable and non-iterable GLAs'); } $glaReqStates[$glaName] = $gla->req_states(); foreach ($gla->req_states() as $rstate) { $reqStates[] = $rstate; } $glaGenStates[$glaName] = $gla->state(); // TODO: Support constant states grokit_assert(!$gla->has_state(), 'Multiplexer currently does not support constant states.'); } $libraries = array_unique($libraries); $extra = ['glas' => $myGLAs]; ?> class <?php echo $className; ?> { <?php foreach ($myGLAs as $name => $type) { ?> <?php echo $type; ?> <?php echo $name; ?> ; <?php } // foreach inner gla ?> class Iterator { bool _gotResultsOnce; bool _valid; <?php foreach ($myGLAs as $name => $type) { ?> <?php echo $type; ?> * it_<?php echo $name; ?> ; <?php } // foreach inner gla ?> public: Iterator(void) : _gotResultsOnce(false), _valid(false), <?php echo array_template('it_{key}(nullptr)', ', ', $myGLAs); ?> { } Iterator(<?php echo typed_ref_args($myGLAs); ?> ) : _gotResultsOnce(false), _valid(true), <?php echo array_template('it_{key}(&{key})', ', ', $myGLAs); ?> { <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'multi') { ?> <?php echo $name; ?> .Finalize(); <?php } // if inner GLA is multi } // foreach inner gla ?> } Iterator( const Iterator & other) = default; ~Iterator() { } bool GetNextResult( <?php echo typed_ref_args($outputs); ?> ) { FATALIF(!_valid, "Tried to get results from an invalid iterator."); bool ret = !_gotResultsOnce; _gotResultsOnce = true; <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'multi') { ?> ret |= it_<?php echo $name; ?> ->GetNextResult(<?php echo args($glaOutputs[$name]); ?> ); <?php } // if inner GLA is multi } // foreach inner gla ?> if( ret ) { <?php foreach ($myGLAs as $name => $type) { if ($glaRez[$name] == 'single') { ?> it_<?php echo $name; ?> ->GetResult(<?php echo args($glaOutputs[$name]); ?> ); <?php } else { if ($glaRez[$name] == 'state') { $stateVar = array_keys($glaOutputs[$name])[0]; $stateType = $glaOutputs[$name][$stateVar]; ?> <?php echo $stateVar; ?> = <?php echo $stateType; ?> (it_<?php echo $name; ?> ); <?php } } // if inner GLA is state } // foreach inner gla ?> } return ret; } }; Iterator multiIterator; public: <?php echo $className; ?> () { } ~<?php echo $className; ?> () { } void AddItem(<?php echo const_typed_ref_args($inputs); ?> ) { // Call AddItem individually on each GLA. <?php foreach ($myGLAs as $gName => $gType) { ?> <?php echo $gName; ?> .AddItem(<?php echo args($glaInputs[$gName]); ?> ); <?php } // foreach inner gla ?> } void AddState( <?php echo $className; ?> & other ) { // Call AddState individually on each GLA. <?php foreach ($myGLAs as $gName => $gType) { ?> <?php echo $gName; ?> .AddState(other.<?php echo $gName; ?> ); <?php } // foreach inner gla ?> } void Finalize() { multiIterator = Iterator(<?php echo args($myGLAs); ?> ); } bool GetNextResult(<?php echo typed_ref_args($outputs); ?> ) { return multiIterator.GetNextResult(<?php echo args($outputs); ?> ); } void GetResult(<?php echo typed_ref_args($outputs); ?> ) { Finalize(); GetNextResult(<?php echo args($outputs); ?> ); } <?php foreach (array_keys($myGLAs) as $index => $name) { ?> const <?php echo $myGLAs[$name]; ?> & GetGLA<?php echo $index; ?> () const { return <?php echo $name; ?> ; } <?php } ?> }; <?php return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => $resultType, 'libraries' => $libraries, 'configurable' => $configurable, 'extra' => $extra); }
?> ; } <?php $gContent .= ob_get_clean(); ?> ////////////// // Operators // all operators defined by C++ <?php return array('kind' => 'TYPE', "system_headers" => array("cstdlib", "cstdio", "cinttypes", 'limits'), "complex" => false, "global_content" => $gContent, 'binary_operators' => ['+', '-', '*', '/', '%', '==', '!=', '>', '<', '>=', '<='], 'unary_operators' => ['+', '-'], 'constructors' => $constructors, 'functions' => $functions, 'properties' => ['integral', 'numeric', '_primative_'], 'describe_json' => DescribeJson('integer'), 'extras' => ['size.bytes' => 2]); } declareOperator('//', ['base::SMALLINT', 'base::SMALLINT'], function ($args) { $l = lookupType('base::SMALLINT'); return ['kind' => 'OPERATOR', 'name' => '/', 'input' => [$l, $l], 'result' => $l, 'deterministic' => true]; }); // Define the constructors from the various other types. foreach (['BYTE', 'SMALLINT', 'INT', 'BIGINT', 'FLOAT', 'DOUBLE'] as $type) { $fullType = 'base::' . $type; $call = function ($args, $targs = []) use($fullType) { $arg = lookupType($fullType); $ret = lookupType('base::SMALLINT'); return array('kind' => 'FUNCTION', 'input' => [$arg], 'result' => $ret, 'deterministic' => true); }; declareFunctionGlobal('base', 'SMALLINT', [$fullType], $call); } declareSynonym("base::SMALL", "base::SMALLINT");
function Average(array $t_args, array $input, array $output) { $className = generate_name('Average'); grokit_assert(\count($input) == \count($output), 'Average must have the same number of inputs and outputs'); $outToIn = []; $internalTypes = []; $internalInit = []; reset($output); foreach ($input as $name => $type) { $outKey = key($output); $outToIn[$outKey] = $name; if ($type->is('numeric')) { $internalTypes[$name] = 'long double'; $internalInit[$name] = '0.0'; } else { $internalTypes[$name] = $type; $internalInit[$name] = ''; } if (is_null(current($output))) { if ($type->is('numeric')) { $output[$outKey] = lookupType('base::DOUBLE'); } else { $output[$outKey] = $type; } } next($output); } $countType = 'uint64_t'; $debug = get_default($t_args, 'debug', 0); ?> class <?php echo $className; ?> { private: <?php echo $countType; ?> count; // keeps the number of tuples aggregated <?php foreach ($internalTypes as $name => $type) { ?> <?php echo $type; ?> sum_<?php echo $name; ?> ; <?php } // foreach internal value ?> public: <?php echo $className; ?> () : count(0) <?php foreach ($internalInit as $name => $init) { ?> , sum_<?php echo $name; ?> (<?php echo $init; ?> ) <?php } // foreach internal initializer ?> {} void AddItem(<?php echo const_typed_ref_args($input); ?> ) { count++; <?php foreach ($input as $name => $type) { ?> sum_<?php echo $name; ?> += <?php echo $name; ?> ; <?php } // foreach input ?> } void AddState(<?php echo $className; ?> & o){ count += o.count; <?php foreach ($input as $name => $type) { ?> sum_<?php echo $name; ?> += o.sum_<?php echo $name; ?> ; <?php } // foreach input ?> } // we only support one tuple as output void GetResult(<?php echo typed_ref_args($output); ?> ) const { if( count > 0 ) { <?php foreach ($output as $name => $type) { $inName = $outToIn[$name]; ?> <?php echo $name; ?> = (sum_<?php echo $inName; ?> ) / count; <?php } // foreach output ?> } else { <?php foreach ($output as $name => $type) { ?> <?php echo $name; ?> = sum_<?php echo $inName; ?> ; <?php } // foreach output ?> } } }; <?php $sys_headers = ['cinttypes']; if ($debug > 0) { $sys_headers[] = 'iostream'; $sys_headers[] = 'sstream'; } return array('kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'input' => $input, 'output' => $output, 'result_type' => 'single'); }
function GroupBy(array $t_args, array $inputs, array $outputs, array $states) { // Ensure we have valid inputs. if (\count($inputs) == 0) { // No inputs given, try to get them from template arguments. grokit_assert(array_key_exists('input', $t_args), 'No inputs given for GroupBy'); $inputs = $t_args['input']; if (!is_array($inputs)) { $inputs = [$inputs]; } foreach ($inputs as $name => &$type) { if (is_identifier($type)) { $type = lookupType(strval($type)); } grokit_assert(is_datatype($type), 'Invalid type given for input ' . $name); } } grokit_assert(array_key_exists('group', $t_args), 'No groups specified for GroupBy'); $gbyAttMap = $t_args['group']; grokit_assert(is_array($gbyAttMap), 'Invalid value given for groups, expected an expression name or list of expression names'); $gbyAttMap = array_map('strval', $gbyAttMap); $gbyAttNames = array_keys($gbyAttMap); foreach ($gbyAttMap as $in => $out) { grokit_assert(array_key_exists($in, $inputs), 'Group ' . $in . ' not present in input'); grokit_assert(array_key_exists($out, $outputs), 'Output Attribute ' . $out . ' for group ' . $in . ' not found in outputs'); } $numGByAtts = \count($gbyAttNames); grokit_assert(array_key_exists('aggregate', $t_args), 'No aggregate specified for GroupBy'); $innerGLA = $t_args['aggregate']; grokit_assert(is_gla($innerGLA), 'Non-GLA specified as aggregate for GroupBy'); $debug = get_default($t_args, 'debug', 0); $init_size = get_default($t_args, 'init.size', 1024); $use_mct = get_default($t_args, 'use.mct', true); $keepHashes = get_default($t_args, 'mct.keep.hashes', false); grokit_assert(is_bool($keepHashes), 'GroupBy mct.keep.hashes argument must be boolean'); // determine the result type $use_fragments = get_default($t_args, 'use.fragments', true); $resType = $use_fragments ? ['fragment', 'multi'] : ['multi']; $fragSize = get_default($t_args, 'fragment.size', 2000000); // Always support state $resType[] = 'state'; // Class name randomly generated $className = generate_name("GroupBy"); // instantiate the inner GLA. input/output is derived from the main input/output $gbyAtts = []; $gbyAttsOut = []; $glaInputAtts = []; $glaOutputAtts = []; foreach ($inputs as $name => $type) { if (in_array($name, $gbyAttNames)) { $gbyAtts[$name] = $type; $gbyAttsOut[$gbyAttMap[$name]] = $type; $outputs[$gbyAttMap[$name]] = $type; } else { $glaInputAtts[$name] = $type; } } foreach ($outputs as $name => $type) { if (!in_array($name, $gbyAttMap)) { $glaOutputAtts[$name] = $type; } } $innerGLA = $innerGLA->apply($glaInputAtts, $glaOutputAtts, $states); $libraries = $innerGLA->libraries(); $innerRes = get_first_value($innerGLA->result_type(), ['multi', 'single', 'state']); if ($innerRes == 'state') { // If the result type is state, the only output is a state object // containing the GLA. $outputName = array_keys($glaOutputAtts)[0]; $innerOutputs = [$outputName => lookupType('base::STATE', ['type' => $innerGLA])]; } else { $innerOutputs = $innerGLA->output(); grokit_assert(\count($innerOutputs) == \count($glaOutputAtts), 'Expected ' . \count($glaOutputAtts) . ' outputs fromm Inner GLA, got ' . \count($innerOutputs)); } $constState = lookupResource('GroupByState', ['gla' => $innerGLA, 'groups' => $gbyAtts, 'debug' => $debug]); // constructor argumetns are inherited from inner GLA $configurable = $innerGLA->configurable(); $reqStates = $innerGLA->req_states(); // We need to specially create the constructor string because apparently // declaring Type Name(); is a function declaration instead of a variable // declaration for some reason. $constructorParts = []; if ($configurable) { $constructorParts[] = 'jsonInit'; } if ($innerGLA->has_state()) { $constructorParts[] = 'innerState'; } $constructorString = \count($constructorParts) > 0 ? '(' . implode(', ', $constructorParts) . ')' : ''; // add the outputs we got from the gla foreach ($innerOutputs as $name => $type) { grokit_assert(array_key_exists($name, $outputs), 'Inner GLA\'s outputs refer to unknown attribute ' . $name); grokit_assert($type !== null, 'GroupBy Inner GLA left output ' . $name . ' with no type'); $outputs[$name] = $type; } $iterable = $innerGLA->iterable(); // need to keep track of system includes needed $extraHeaders = array(); $allocatorText = "std::allocator<std::pair<const Key, {$innerGLA}> >"; if ($use_mct) { $keepHashesText = $keepHashes ? 'true' : 'false'; $extraHeaders[] = "mct/hash-map.hpp"; $map = "mct::closed_hash_map<Key, {$innerGLA}, HashKey, std::equal_to<Key>, {$allocatorText}, {$keepHashesText}>"; $mapType = 'mct::closed_hash_map'; } else { $extraHeaders[] = "unordered_map"; $map = "std::unordered_map<Key, {$innerGLA}, HashKey, std::equal_to<Key>, {$allocatorText}>"; $mapType = 'std::unordered_map'; } if ($debug > 0) { $extraHeaders[] = 'cstdio'; } ?> class <?php echo $className; ?> { public: using ConstantState = <?php echo $constState; ?> ; <?php if ($innerGLA->has_state()) { ?> using InnerState = ConstantState::InnerState; <?php } // if gla has state ?> using Key = ConstantState::Key; using HashKey = ConstantState::HashKey; using InnerGLA = <?php echo $innerGLA; ?> ; typedef <?php echo $map; ?> MapType; static const size_t INIT_SIZE = <?php echo $init_size; ?> ; public: class Iterator { MapType::iterator it; // current value MapType::iterator end; // last value in the fragment public: Iterator() { } Iterator(MapType::iterator _it, MapType::iterator _end): it(_it), end(_end) { if( it != end ) { <?php switch ($innerRes) { case 'multi': ?> it->second.Finalize(); <?php break; case 'state': if ($innerGLA->finalize_as_state()) { ?> it->second.FinalizeState(); <?php } // if we need to finalize as a state break; } // end switch inner restype ?> } } bool GetNextResult( <?php echo typed_ref_args($outputs); ?> ) { bool gotResult = false; while( it != end && !gotResult ) { <?php echo $innerGLA; ?> & gla = it->second; <?php foreach ($gbyAttMap as $in => $out) { ?> <?php echo $out; ?> = it->first.<?php echo $in; ?> ; <?php } // foreach grouping attribute ?> <?php switch ($innerRes) { case 'multi': ?> gotResult = gla.GetNextResult( <?php echo args($innerOutputs); ?> ); if( !gotResult ) { ++it; if( it != end ) { it->second.Finalize(); } } <?php break; case 'single': ?> gotResult = true; gla.GetResult(<?php echo args($innerOutputs); ?> ); ++it; <?php break; case 'state': reset($innerOutputs); // Assuming that $innerOutputs contains a single value that is // the state type. $oName = key($innerOutputs); $oType = current($innerOutputs); ?> gotResult = true; <?php echo $oName; ?> = <?php echo $oType; ?> ( &gla ); ++it; <?php } // switch inner result type ?> } return gotResult; } }; private: const ConstantState & constState; <?php if ($configurable) { ?> const Json::Value jsonInit; <?php } // if configurable ?> size_t count; MapType groupByMap; std::vector<MapType::iterator> theIterators; // the iterators, only 2 elements if multi, many if fragment Iterator multiIterator; public: <?php echo $className; ?> (<?php if ($configurable) { ?> const Json::Value & _jsonInit, <?php } ?> const ConstantState & _constState ) : constState(_constState) <?php if ($configurable) { ?> , jsonInit(_jsonInit) <?php } // if configurable ?> , count(0) , groupByMap( INIT_SIZE ) , theIterators() , multiIterator() { } ~<?php echo $className; ?> () {} void Reset(void) { count = 0; groupByMap.clear(); theIterators.clear(); } void AddItem(<?php echo array_template('const {val} & {key}', ', ', $inputs); ?> ) { count++; // check if _key is already in the map; if yes, add _value; else, add a new // entry (_key, _value) Key key(<?php echo array_template('{key}', ', ', $gbyAtts); ?> ); MapType::iterator it = groupByMap.find(key); if (it == groupByMap.end()) { // group does not exist // create an empty GLA and insert // better to not add the item here so we do not have // to transport a large state <?php if ($innerGLA->has_state()) { ?> const InnerState & innerState = constState.getConstState(key); <?php } // if gla has state ?> InnerGLA gla<?php echo $constructorString; ?> ; auto ret = groupByMap.insert(MapType::value_type(key, gla)); it = ret.first; // reposition } it->second.AddItem(<?php echo array_template('{key}', ', ', $glaInputAtts); ?> ); } void AddState(<?php echo $className; ?> & other) { count += other.count; // scan other hash and insert or update content in this one for (MapType::iterator it = other.groupByMap.begin(); it != other.groupByMap.end(); ++it) { const Key& okey = it->first; <?php echo $innerGLA; ?> & ogla = it->second; MapType::iterator itt = groupByMap.find(okey); if (itt != groupByMap.end()) { // found the group <?php echo $innerGLA; ?> & gla = itt->second; gla.AddState(ogla); } else { // add the other group to this hash groupByMap.insert(MapType::value_type(okey, ogla)); } } } <?php if ($iterable) { ?> bool ShouldIterate(ConstantState& modibleState) { <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : ==== ShouldIterate ====\n"); <?php } // if debugging enabled ?> bool shouldIterate = false; for( MapType::iterator it = groupByMap.begin(); it != groupByMap.end(); ++it ) { const Key & key = it->first; InnerGLA & gla = it->second; <?php if ($innerGLA->has_state()) { ?> InnerState & innerState = modibleState.getModibleState(key); <?php } // if gla has state ?> bool glaRet = gla.ShouldIterate(innerState); shouldIterate = shouldIterate || glaRet; <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : Key(%s) shouldIterate(%s)\n", key.to_string().c_str(), glaRet ? "true" : "false"); <?php } // if debugging enabled ?> } return shouldIterate; } <?php } // if iterable ?> <?php if (in_array('fragment', $resType)) { ?> int GetNumFragments(void){ int size = groupByMap.size(); int sizeFrag = <?php echo $fragSize; ?> ; // setup the fragment boundaries // scan via iterator and count int frag=0; int pos=0; MapType::iterator it = groupByMap.begin(); theIterators.clear(); theIterators.push_back( it ); // special case when size < num_fragments // > if (sizeFrag == 0){ it = groupByMap.end(); theIterators.push_back( it ); return 1; // one fragment } while(it!=groupByMap.end()){ while(it!=groupByMap.end() && pos<( frag + 1 )*sizeFrag){ //> ++it; pos++; } theIterators.push_back( it ); frag++; } <?php if ($debug > 0) { ?> fprintf(stderr, "<?php echo $className; ?> : fragments(%d)\n", frag); <?php } ?> return frag; } Iterator* Finalize(int fragment){ // Call finalize on all inner GLAs in this fragment. MapType::iterator iter = theIterators[fragment]; MapType::iterator iterEnd = theIterators[fragment+1]; Iterator* rez = new Iterator(theIterators[fragment], theIterators[fragment+1] ); return rez; } bool GetNextResult(Iterator* it, <?php echo array_template('{val} & {key}', ', ', $outputs); ?> ) { return it->GetNextResult(<?php echo args($outputs); ?> ); } <?php } // if using fragment interface ?> void Finalize() { multiIterator = Iterator( groupByMap.begin(), groupByMap.end() ); <?php if ($debug >= 1) { ?> fprintf(stderr, "<?php echo $className; ?> : groups(%lu) tuples(%lu)\n", groupByMap.size(), count); <?php } ?> } bool GetNextResult(<?php echo array_template('{val} & {key}', ', ', $outputs); ?> ) { return multiIterator.GetNextResult( <?php echo args($outputs); ?> ); } std::size_t size() const { return groupByMap.size(); } const MapType& GetMap() const { return groupByMap; } bool Contains(<?php echo const_typed_ref_args($gbyAtts); ?> ) const { Key key(<?php echo args($gbyAtts); ?> ); return groupByMap.count(key) > 0; } const InnerGLA& Get(<?php echo const_typed_ref_args($gbyAtts); ?> ) const { Key key(<?php echo args($gbyAtts); ?> ); return groupByMap.at(key); } bool Contains(Key key) const { return groupByMap.count(key) > 0; } const InnerGLA& Get(Key key) const { return groupByMap.at(key); } }; <?php if (in_array('fragment', $resType)) { ?> typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php } ?> <?php $sys_headers = array_merge(['iomanip', 'iostream', 'cstring'], $extraHeaders); return array('kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => array('HashFunctions.h'), 'input' => $inputs, 'output' => $outputs, 'result_type' => $resType, 'configurable' => $configurable, 'generated_state' => $constState, 'required_states' => $reqStates, 'iterable' => $iterable, 'properties' => ['resettable', 'finite container'], 'libraries' => $libraries, 'extra' => ['inner_gla' => $innerGLA]); }
?> <?php return ['kind' => 'TYPE', 'complex' => false, 'system_headers' => $system_headers, 'user_headers' => ['Config.h'], 'binary_operators' => $bin_operators, 'global_content' => $globalContent, 'constructors' => $constructors, 'methods' => $methods, 'functions' => $functions, 'libraries' => $libraries, 'properties' => ['clusterable'], 'describe_json' => DescribeJson('datetime', DescribeJsonStatic(['format' => 'YYYY-MM-DD HH:mm:ss'])), 'extras' => ['size.bytes' => 4]]; } // DATETIME + INT = DATETIME declareOperator('+', ['base::DATETIME', 'base::INT'], function ($args, $targs = []) { $dateType = lookupType('base::DATETIME'); $intType = lookupType('base::INT'); return ['kind' => 'OPERATOR', 'input' => [$dateType, $intType], 'result' => $dateType, 'deterministic' => true]; }); // DATETIME - INT = DATETIME declareOperator('-', ['base::DATETIME', 'base::INT'], function ($args, $targs = []) { $dateType = lookupType('base::DATETIME'); $intType = lookupType('base::INT'); return ['kind' => 'OPERATOR', 'input' => [$dateType, $intType], 'result' => $dateType, 'deterministic' => true]; }); // DATETIME - DATETIME = INT declareOperator('-', ['base::DATETIME', 'base::DATETIME'], function ($args, $targs = []) { $dateType = lookupType('base::DATETIME'); $intType = lookupType('base::INT'); return ['kind' => 'OPERATOR', 'input' => [$dateType, $dateType], 'result' => $intType, 'deterministic' => true]; }); foreach (['==', '!=', '<', '>', '<=', '>='] as $op) { declareOperator($op, ['base::DATETIME', 'base::INT'], function ($args, $targs = []) { $dateType = lookupType('base::DATETIME'); $intType = lookupType('base::INT'); $boolType = lookupType('base::BOOL'); return ['kind' => 'OPERATOR', 'input' => [$dateType, $intType], 'result' => $boolType, 'deterministic' => true]; }); }
/** * GI that generates data in clusters, using a specified distribution for each * cluster. * * This GI requires the following template arguments: * - 'n' or 0 * The number of tuples to generate. Note: this value is per task. * The total number of tuples generated will be n_tasks * n * - 'centers' or 1 * A list of configuration for the centers. * * The following template arguments are optional: * - 'outputs' * If the outputs of the GI are not given implicitly, they can be * specified in this template argument. The number of dimensions will * be determined by the number of outputs. * * All output types must be numeric real types. The default type for * outputs is DOUBLE. * - 'dist.lib' = 'std' * Which library to use for generating distributions. * Valid options are: * - std * - boost * - 'seed' = null * The seed to be used for the random number generator. This seed will * be used to generate the seed for each task, and different runs with * the same seed will produce the same data. * - 'compute.sets' = 1 * The number of sets of tuples to compute at once. * * Each center configuration is a functor with the form: * dist_name(args) * * The following distributions are supported: * { Uniform Distributions } * - uniform(a = 0, b = 1) * * { Normal Distributions } * - normal(mean = 0.0, std_dev = 1.0) [ synonyms: gaussian ] * - inverse_gaussian(mean = 1, shape = 1) [ synonyms: inverse_normal ] * * { Bernoulli Distributions } * - binomial(t = 1, p = 0.5) * - negative_binomial(k = 1, p = 0.5) * * { Poisson Distributions } * - exponential( lambda = 1 ) * - gamma(alpha = 1, beta = 1) [ synonyms: Gamma ] */ function ClusterGen(array $t_args, array $outputs) { $sys_headers = ['array', 'cinttypes']; $user_headers = []; $libraries = []; if (\count($outputs) == 0) { grokit_assert(array_key_exists('outputs', $t_args), 'ClusterGen: No outputs specified'); $count = 0; foreach ($t_args['outputs'] as $type) { if (is_identifier($type)) { $type = lookupType($type); } grokit_assert(is_datatype($type), 'ClusterGen: Non data-type ' . $type . ' given as output'); $name = 'output' . $count++; $outputs[$name] = $type; } } foreach ($outputs as $name => &$type) { if (is_null($type)) { $type = lookupType('base::DOUBLE'); } else { grokit_assert($type->is('real'), 'ClusterGen: Non-real datatype ' . $type . ' given as output'); } } $myOutputs = []; foreach ($outputs as $name => $type) { $myOutputs[$name] = $type; } $tSize = \count($outputs); $seed = get_default($t_args, 'seed', null); if ($seed !== null) { grokit_assert(is_int($seed), 'ClusterGen: Seed must be an integer or null.'); } else { $user_headers[] = 'HashFunctions.h'; } $distLib = get_default($t_args, 'dist.lib', 'std'); $distNS = ''; switch ($distLib) { case 'std': $sys_headers[] = 'random'; $distNS = 'std'; break; case 'boost': $sys_headers[] = 'boost/random.hpp'; $distNS = 'boost::random'; $libraries[] = 'boost_random-mt'; if ($seed === null) { // Need random_device $sys_headers[] = 'boost/random/random_device.hpp'; $libraries[] = 'boost_system-mt'; } break; default: grokit_error('ClusterGen: Unknown RNG library ' . $distLib); } $distRNG = 'mt19937'; $RNGtype = $distNS . '::' . $distRNG; $nTuples = get_first_key($t_args, ['n', '0']); grokit_assert(is_int($nTuples), 'ClusterGen: the number of tuples to be produced must be an integer.'); $centers = get_first_key($t_args, ['centers', 1]); grokit_assert(is_array($centers), 'ClusterGen: centers must be an array of functors'); $handleDist = function ($name, $args, $oType) use($distNS) { $distName = ''; $distArgs = []; switch ($name) { case 'gaussian': case 'normal': $distName = $distNS . '::' . 'normal_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Normal distribution takes at most 2 arguments, ' . \count($args) . ' given'); $mean = get_default($args, ['mean', 0], 0.0); $sigma = get_default($args, ['std_dev', 'sigma', 1], 1.0); grokit_assert(is_numeric($mean), 'ClusterGen: mean parameter of binomial distribution must be a real number.'); grokit_assert(is_numeric($sigma), 'ClusterGen: sigma parameter of binomial distribution must be a real number.'); $mean = floatval($mean); $sigma = floatval($sigma); $distArgs = [$mean, $sigma]; break; case 'binomial': $distName = $distNS . '::' . 'binomial_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Binomial distribution takes at most 2 arguments, ' . \count($args) . ' given'); $t = get_default($args, ['t', 0], 1); $p = get_default($args, ['p', 1], 0.5); grokit_assert(is_int($t), 'ClusterGen: t parameter of binomial distribution must be an integer.'); grokit_assert(is_numeric($p), 'ClusterGen: p parameter of binomial distribution must be a real number.'); $p = floatval($p); grokit_assert($p >= 0 && $p <= 1, 'ClusterGen: p parameter of binomial distribution must be in the range [0, 1]'); grokit_assert($t >= 0, 'ClusterGen: t parameter of binomial distribution must be in the range [0, +inf)'); $distArgs = [$t, $p]; break; case 'negative_binomial': $distName = $distNS . '::' . 'negative_binomial_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Negative Binomial distribution takes at most 2 arguments, ' . \count($args) . ' given'); $k = get_default($args, ['k', 0], 1); $p = get_default($args, ['p', 1], 0.5); grokit_assert(is_int($k), 'ClusterGen: k parameter of binomial distribution must be an integer.'); grokit_assert(is_numeric($p), 'ClusterGen: p parameter of binomial distribution must be a real number.'); $p = floatval($p); grokit_assert($p > 0 && $p <= 1, 'ClusterGen: p parameter of negative binomial distribution must be in the range (0, 1]'); grokit_assert($k > 0, 'ClusterGen: k parameter of negative binomial distribution must be in the range (0, +inf)'); $distArgs = [$k, $p]; break; case 'inverse_gaussian': case 'inverse_normal': grokit_assert(\count($args) <= 2, 'ClusterGen: Inverse Gaussian distribution takes at most 2 arguments, ' . \count($args) . ' given'); $mean = get_default($args, ['mean', 0], 1); $shape = get_default($args, ['shape', 1], 1); grokit_assert(is_numeric($mean), 'ClusterGen: mean parameter of inverse gaussian distribution must be a real number.'); grokit_assert(is_numeric($shape), 'ClusterGen: shape parameter of inverse gaussian distribution must be a real number.'); $mean = floatval($mean); $shape = floatval($shape); grokit_assert($mean > 0, 'ClusterGen: mean of inverse gaussian distribution must be in range (0, inf)'); grokit_assert($shape > 0, 'ClusterGen: shape of inverse gaussian distribution must be in range (0, inf)'); $gen_args = ['output' => $oType, 'ns' => $distNS]; $distName = strval(lookupResource('datagen::InverseGaussianGen', $gen_args)); $distArgs = [$mean, $shape]; break; case 'uniform': $distName = $distNS . '::' . 'uniform_real_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Uniform distribution takes at most 2 arguments, ' . \count($args) . ' given'); $a = get_default($args, ['a', 0], 0.0); $b = get_default($args, ['b', 1], 1.0); grokit_assert(is_numeric($a), 'ClusterGen: `a` parameter of uniform distribution must be a real number.'); grokit_assert(is_numeric($b), 'ClusterGen: `b` parameter of uniform distribution must be a real number.'); $a = floatval($a); $b = floatval($b); grokit_assert($b >= $a, 'ClusterGen: `b` parameter of uniform distribution must be >= the `a` parameter.'); $distArgs = [$a, $b]; break; case 'exponential': $distName = $distNS . '::' . 'exponential_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 1, 'ClusterGen: Exponential distribution takes at most 1 argument.'); $lambda = get_default($args, ['lambda', 0], 1.0); grokit_assert(is_numeric($lambda), 'ClusterGen: `lambda` parameter of exponential distribution must be a real number.'); $lambda = floatval($lambda); grokit_assert($lambda > 0, 'ClusterGen: `lambda` parameter of exponential distribution must be in range (0, +inf).'); $distArgs = [$lambda]; break; case 'gamma': case 'Gamma': $distName = $distNS . '::' . 'gamma_distribution<' . $oType . '>'; grokit_assert(\count($args) <= 2, 'ClusterGen: Gamma distribution takes at most 2 arguments.'); $alpha = get_default($args, ['alpha', 0], 1.0); $beta = det_default($args, ['beta', 1], 1.0); grokit_assert(is_numeric($alpha), 'ClusterGen: `alpha` parameter of gamma distribution must be a real number.'); grokit_assert(is_numeric($beta), 'ClusterGen: `beta` parameter of gamma distribution must be a real number.'); $alpha = floatval($alpha); $beta = floatval($beta); $distArgs = [$alpha, $beta]; break; default: grokit_error('ClusterGen: Unknown distribution ' . $name . ' given for center'); } return [$distName, $distArgs]; }; $dists = []; $distArgs = []; $count = 0; $oType = ''; $nCenters = 1; reset($outputs); foreach ($centers as $val) { $cluster = $val; if (is_functor($val)) { $cluster = [$val]; } else { if (is_array($val)) { $nCenters = lcm($nCenters, \count($val)); } else { grokit_error('ClusterGen: center descriptions must be functors or list of functors'); } } $curDist = []; $curDistArgs = []; $curDistName = 'distribution' . $count++; $oType = strval(current($outputs)); $iCount = 0; foreach ($cluster as $functor) { grokit_assert(is_functor($functor), 'ClusterGen: center description must be a functor'); $vName = $curDistName . '_' . $iCount++; $ret = $handleDist($functor->name(), $functor->args(), $oType); $curDist[$vName] = $ret[0]; $curDistArgs[$vName] = $ret[1]; } next($outputs); $dists[$curDistName] = $curDist; $distArgs[$curDistName] = $curDistArgs; } // Determine the default number of sets to compute at a time. // We want to generate either $nTuples or 10,000 tuples, depending on which // is less. $defaultSetsTarget = min($nTuples, 10000); $setsToTarget = intval(ceil($defaultSetsTarget / $nCenters)); $computeSets = get_default($t_args, 'compute.sets', $setsToTarget); grokit_assert(is_int($computeSets) && $computeSets > 0, 'ClusterGen: compute.sets must be a positive integer, ' . $computeSets . ' given'); $className = generate_name('ClusterGen'); // For some BIZZARE reason, the $outputs array was getting modified while // traversing over the $dists array. Making a deep copy of the outputs and // then reassigning it seems to fix the issue. $outputs = $myOutputs; ?> class <?php echo $className; ?> { // The number of tuples to produce per task static constexpr size_t N = <?php echo $nTuples; ?> ; static constexpr size_t CacheSize = <?php echo $computeSets * $nCenters; ?> ; // Typedefs typedef std::tuple<<?php echo array_template('{val}', ', ', $outputs); ?> > Tuple; typedef std::array<Tuple, CacheSize> TupleArray; typedef TupleArray::const_iterator TupleIterator; typedef <?php echo $RNGtype; ?> RandGen; // Number of tuples produced. uintmax_t count; // Cache a number of outputs for efficiency TupleArray cache; TupleIterator cacheIt; // Random number generator RandGen rng; // Distributions <?php // This is the section causing issues. foreach ($dists as $name => $list) { foreach ($list as $vName => $type) { ?> <?php echo $type; ?> <?php echo $vName; ?> ; <?php } // foreach distribution } // foreach cluster set ?> // Helper function to generate tuples. void GenerateTuples(void) { <?php $tIndex = 0; foreach ($dists as $name => $list) { $lCenters = \count($list); // $nCenters has been defined to be the LCM of the number of centers in // any column, so $lCenter is guaranteed to divide evenly into // CacheSize ?> for( size_t index = 0; CacheSize > index; index += <?php echo $lCenters; ?> ) { <?php $index = 0; foreach ($list as $vName => $type) { ?> std::get<<?php echo $tIndex; ?> >(cache[index + <?php echo $index; ?> ]) = <?php echo $vName; ?> (rng); <?php $index++; } // foreach value in tuple ?> } <?php $tIndex++; } // foreach distribution ?> cacheIt = cache.cbegin(); } public: // Constructor <?php echo $className; ?> ( GIStreamProxy & _stream ) : cache() , cacheIt() , count(0) , rng() <?php foreach ($dists as $name => $list) { foreach ($list as $vName => $type) { ?> , <?php echo $vName; ?> (<?php echo implode(', ', $distArgs[$name][$vName]); ?> ) <?php } // foreach distribution } // foreach cluster set ?> { <?php if (is_null($seed)) { ?> <?php echo $distNS; ?> ::random_device rd; <?php } // if seed is null ?> RandGen::result_type seed = <?php echo is_null($seed) ? 'rd()' : "CongruentHash({$seed}, _stream.get_id() )"; ?> ; rng.seed(seed); cacheIt = cache.cend(); } // Destructor ~<?php echo $className; ?> (void) { } bool ProduceTuple(<?php echo typed_ref_args($outputs); ?> ) { if( N > count ) { if( cacheIt == cache.cend() ) { GenerateTuples(); } <?php $tIndex = 0; foreach ($outputs as $name => $type) { ?> <?php echo $name; ?> = std::get<<?php echo $tIndex; ?> >(*cacheIt); <?php $tIndex++; } // foreach output ?> ++cacheIt; ++count; return true; } else { return false; } } }; <?php return array('kind' => 'GI', 'name' => $className, 'output' => $outputs, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'libraries' => $libraries); }
function Sum(array $t_args, array $inputs, array $outputs) { $className = generate_name("Sum"); $storage = []; $inits = []; if (\count($inputs) == 0) { $inputs = ["x" => lookupType("base::DOUBLE")]; $storage = ["x" => 'long double']; $inits = ["x" => '']; $outputs = $inputs; } else { $oInputs = $inputs; reset($outputs); foreach ($oInputs as $name => $value) { if ($value->is('real')) { $storage[$name] = 'long double'; } else { if ($value->is('integral')) { $storage[$name] = 'long long int'; } else { if ($value == lookupType('base::BOOL')) { $storage[$name] = 'long int'; } else { $storage[$name] = $value->value(); } } } $oKey = key($outputs); if ($outputs[$oKey] === null) { if ($value->is('real')) { $outputs[$oKey] = lookupType('base::DOUBLE'); } else { if ($value->is('integral') || $value == lookupType('base::BOOL')) { $outputs[$oKey] = lookupType('base::BIGINT'); } else { $outputs[$oKey] = $value; } } } $inits[$name] = $value->has('init') ? $value->get('init') : ''; next($outputs); } } ?> class <?php echo $className; ?> { <?php echo array_template('{val} {key};' . PHP_EOL, ' ', $storage); ?> public: <?php echo $className; ?> () : <?php echo array_template('{key}({val})', ', ', $inits); ?> { } void AddItem(<?php echo array_template('const {val}& _{key}', ', ', $inputs); ?> ) { <?php echo array_template('{key} += _{key};' . PHP_EOL, ' ', $inputs); ?> } void AddState( <?php echo $className; ?> & other ) { <?php echo array_template('{key} += other.{key};' . PHP_EOL, ' ', $inputs); ?> } void GetResult( <?php echo array_template('{val}& _{key}', ', ', $outputs); ?> ) const { <?php reset($outputs); reset($inputs); foreach ($outputs as $name => $type) { $inName = key($inputs); ?> _<?php echo $name; ?> = <?php echo $inName; ?> ; <?php next($inputs); } ?> } }; <?php return array('kind' => 'GLA', 'name' => $className, 'input' => $inputs, 'output' => $outputs, 'result_type' => 'single'); }
function CCompUnionFindLPFrag($t_args, $inputs, $outputs) { // Class name is randomly generated. $className = generate_name('CCompUnionFindLPFrag'); // Initializiation of argument names. $inputs_ = array_combine(['s', 't'], $inputs); $vertex = $inputs_['s']; // Construction of outputs. $outputs_ = ['node' => $vertex, 'component' => lookupType('base::BIGINT')]; $outputs = array_combine(array_keys($outputs), $outputs_); $sys_headers = ['armadillo', 'algorithm']; $user_headers = []; $lib_headers = []; $libraries = ['armadillo']; $properties = []; $extra = []; $result_type = ['fragment']; ?> using namespace arma; using namespace std; class <?php echo $className; ?> ; <?php $constantState = lookupResource('graph::CCompUnionFindLPFrag_Constant_State', ['className' => $className]); ?> class <?php echo $className; ?> { public: // The constant state for this GLA. using ConstantState = <?php echo $constantState; ?> ; // The current and final indices of the result for the given fragment. using Iterator = std::pair<uint64_t, uint64_t>; // The number of iterations to perform, not counting the initial set-up. static const constexpr int kIterations = 10; // The work is split into chunks of this size before being partitioned. static const constexpr int kBlock = 32; // The maximum number of fragments to use. static const constexpr int kMaxFragments = 64; private: // Node Component static arma::rowvec node_component; // keep track of component size static arma::rowvec component_size; // The typical constant state for an iterable GLA. const ConstantState& constant_state; // The number of unique nodes seen. uint64_t num_nodes; // The current iteration. int iteration; // The number of fragmetns for the result. int num_fragments; // check if need more iterations uint64_t connections; public: <?php echo $className; ?> (const <?php echo $constantState; ?> & state) : constant_state(state), num_nodes(state.num_nodes), iteration(state.iteration),connections(0) { } uint64_t Find(uint64_t node_id){ // use path compression here while (node_id != node_component(node_id)){ node_component(node_id) = node_component(node_component(node_id)); node_id = node_component(node_id); } return node_id; } void Union(uint64_t pid, uint64_t qid){ // find their root pid = Find(pid); qid = Find(qid); if (pid == qid) return; ++ connections; uint64_t psz = component_size(pid), qsz = component_size(qid); if (psz > qsz){ node_component(qid) = pid; component_size(pid) += qsz; }else{ node_component(pid) = qid; component_size(qid) += psz; } } // Basic dynamic array allocation. void AddItem(<?php echo const_typed_ref_args($inputs_); ?> ) { if (iteration == 0) { num_nodes = max((uint64_t) max(s, t), num_nodes); return; } else{ Union((uint64_t) s, (uint64_t) t); } } // Hashes are merged. void AddState(<?php echo $className; ?> &other) { if (iteration == 0) num_nodes = max(num_nodes, other.num_nodes); else connections += other.connections; } // Most computation that happens at the end of each iteration is parallelized // by performed it inside Finalize. bool ShouldIterate(ConstantState& state) { state.iteration = ++iteration; printf("Entering ShouldIterate. connections: %lu, iteration: %d\n", connections, iteration); if (iteration == 1) {// allocate memory // num_nodes is incremented because IDs are 0-based. state.num_nodes = ++num_nodes; // Allocating space can't be parallelized. node_component.set_size(num_nodes); for (uint64_t i = 0; i < num_nodes; ++ i){ node_component(i) = i; } component_size.set_size(num_nodes); component_size.fill(1); return true; } else { return connections > 0 && iteration < kIterations + 1; } } int GetNumFragments() { uint64_t size = (num_nodes - 1) / kBlock + 1; // num_nodes / kBlock rounded up. num_fragments = (iteration == 0) ? 0 : min(size, (uint64_t) kMaxFragments); printf("num_nodes: %lu, size: %lu, num_fragments: %d\n", num_nodes, size, num_fragments); return num_fragments; } // Finalize does nothing Iterator* Finalize(int fragment) { uint64_t count = num_nodes; // The ordering of operations is important. Don't change it. uint64_t first = fragment * (count / kBlock) / num_fragments * kBlock; uint64_t final = (fragment == num_fragments - 1) ? count - 1 : (fragment + 1) * (count / kBlock) / num_fragments * kBlock - 1; printf("fragment: %lu\tcount: %lu\tfirst: %lu\tfinal: %lu\n", fragment, count, first, final); return new Iterator(first, final); } bool GetNextResult(Iterator* it, <?php echo typed_ref_args($outputs_); ?> ) { // should iterate is true if (connections > 0 && iteration < kIterations + 1){ printf("I need more iterations, because connections: %lu, iteration: %d\n", connections, iteration); return false; } if (it->first > it->second) return false; node = it->first++; component = Find(node); return true; } }; // Initialize the static member types. arma::rowvec <?php echo $className; ?> ::node_component; arma::rowvec <?php echo $className; ?> ::component_size; typedef <?php echo $className; ?> ::Iterator <?php echo $className; ?> _Iterator; <?php return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $sys_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'properties' => $properties, 'extra' => $extra, 'iterable' => true, 'input' => $inputs, 'output' => $outputs, 'result_type' => $result_type, 'generated_state' => $constantState]; }
<?php $gContent .= ob_get_clean(); ?> ////////////// // Operators // all operators defined by C++ <?php return array('kind' => 'TYPE', "system_headers" => array("cstdlib", "cstdio", "cinttypes", 'limits'), "complex" => false, "global_content" => $gContent, 'binary_operators' => ['+', '-', '*', '/', '%', '==', '!=', '>', '<', '>=', '<='], 'unary_operators' => ['+', '-'], 'constructors' => $constructors, 'functions' => $functions, 'properties' => ['integral', 'numeric', '_primative_'], 'describe_json' => DescribeJson('integer'), 'extras' => ['size.bytes' => 8]); } // end of function declareOperator('//', ['base::BIGINT', 'base::BIGINT'], function ($args) { $l = lookupType('base::BIGINT'); return ['kind' => 'OPERATOR', 'name' => '/', 'input' => [$l, $l], 'result' => $l, 'deterministic' => true]; }); // Define the constructors from the various other types. foreach (['BYTE', 'SMALLINT', 'INT', 'BIGINT', 'FLOAT', 'DOUBLE'] as $type) { $fullType = 'base::' . $type; $call = function ($args, $targs = []) use($fullType) { $arg = lookupType($fullType); $ret = lookupType('base::BIGINT'); return array('kind' => 'FUNCTION', 'input' => [$arg], 'result' => $ret, 'deterministic' => true); }; declareFunctionGlobal('base', 'BIGINT', [$fullType], $call); } ?>