function JoinLHS($wpName, $jDesc)
{
$rhsAttOrder = [];
foreach ($jDesc->hash_RHS_attr as $attr) {
$att = lookupAttribute($attr);
$rhsAttOrder[$att->slot()] = $attr;
}
ksort($rhsAttOrder);
$jDesc->hash_RHS_attr = $rhsAttOrder;
?>
//+{"kind":"WPF", "name":"LHS Lookup", "action":"start"}
extern "C"
int JoinLHSWorkFunc_<?php
echo $wpName;
?>
(WorkDescription &workDescription, ExecEngineData &result) {
double start_time = global_clock.GetTime();
PROFILING2_START;
// this is the area where all of the intermediate, serialized records are stored
SerializedSegmentArray serializedSegments [NUM_SEGS];
// this is the area where all of the records are serialized to;
// 10K bytes are initially used for this
char *serializeHere = (char *) malloc (10000);
// this is the output chunk
Chunk output;
// go to the work description and get the input chunk
JoinLHSWorkDescription myWork;
myWork.swap (workDescription);
Chunk &input = myWork.get_chunkToProcess ();
// get the waypoint ID from the chunk
int wayPointID = myWork.get_wayPointID ();
QueryIDSet queriesToRun = QueryExitsToQueries(myWork.get_whichQueryExits ());
<?php
cgAccessColumns($jDesc->attribute_queries_LHS, 'input', $wpName);
?>
BStringIterator myInBStringIter;
input.SwapBitmap (myInBStringIter);
// start the iterators for the output columns for LHS; used only if stillShallow = 0
<?php
foreach ($jDesc->attribute_queries_LHS_copy as $att => $queries) {
?>
<?php
echo attIteratorType($att);
?>
<?php
echo $att;
?>
_Column_Out;
<?php
}
/*foreach*/
?>
// these manage the output columns that come from the RHS (now stored in the hash table)
<?php
cgConstructColumns(array_keys($jDesc->attribute_queries_RHS_copy));
?>
// this is the ouput bitstring
MMappedStorage myStore;
Column bitmapOut (myStore);
BStringIterator myOutBStringIter (bitmapOut, queriesToRun);
// now we extract all of the hash table segments... after this, myEntries will hold them all
HashTableView myView;
myWork.get_centralHashTable ().EnterReader (myView);
HashTableSegment myEntries[NUM_SEGS];
myView.ExtractAllSegments (myEntries);
// this tells us that we are "still shallow"---not making a deep copy of the LHS atts to the output
int stillShallow = 1;
// the bitstring that will be exracted from the hash table
QueryIDSet *bitstringRHS = 0;
QueryIDSet existsTarget(<?php
echo $jDesc->exists_target;
?>
, true);
QueryIDSet notExistsTarget(<?php
echo $jDesc->not_exists_target;
?>
, true);
// these are all of the attribute values that come from the hash table...
// for each att we need a pointer as well as a dummy value that the pointer will be set to by default
<?php
foreach ($jDesc->attribute_queries_RHS as $att => $queries) {
?>
QueryIDSet <?php
echo attQrys($att);
?>
_RHS(<?php
echo $queries;
?>
, true);
<?php
echo attType($att);
?>
<?php
echo $att;
?>
RHSShadow;
<?php
echo attType($att);
?>
*<?php
echo $att;
?>
RHS = NULL;
<?php
echo attType($att);
?>
<?php
echo $att;
?>
RHSobj;
<?php
}
/*foreach*/
?>
// now actually try to match up all of the tuples!
int totalNum = 0;
while (!myInBStringIter.AtEndOfColumn ()) { // TBD, probably this is not working TBD
// counts how many matches for this query
int numHits = 0;
// extract values of attributes from streams
// now go through the LHS input atts one at a time and extract if it is needed by an active query
// see which queries match up
QueryIDSet curBits = myInBStringIter.GetCurrent ();
curBits.Intersect (queriesToRun);
QueryIDSet exists; // keeps track of the queries for which a match is found
QueryIDSet oldBitstringLHS; // last value of bistringLHS
// if the input query is not empty
if (!curBits.IsEmpty ()) {
totalNum++;
// compute the hash for LHS
HT_INDEX_TYPE hashValue = HASH_INIT;
<?php
foreach ($jDesc->LHS_keys as $att) {
?>
hashValue = CongruentHash(Hash(<?php
echo $att;
?>
_Column.GetCurrent()), hashValue);
<?php
}
/*foreach*/
?>
// figure out which of the hash buckets it goes into
unsigned int index = WHICH_SEGMENT (hashValue);
// now, go to that index and extract matching tuples!
HT_INDEX_TYPE curSlot = WHICH_SLOT (hashValue);
hashValue = curSlot;
// this loops through all of the possible RHS hits
while (1) {
// this is the bitstring that will go in the output
QueryIDSet bitstringLHS;
// for safety (in case we look at a bitstring that spans multiple
// entries that is not done being written by a concurrent writer)
// empty out the inital bitstring
((QueryIDSet *) serializeHere)->Empty ();
// give safe "shadow" values to all of the RHS attributes
<?php
foreach ($jDesc->hash_RHS_attr as $att) {
?>
<?php
echo $att;
?>
RHS = &<?php
echo $att;
?>
RHSShadow;
<?php
}
/*foreach*/
?>
// here we go through and extract the atts one at a time from the hash
// table. Note that the atts must be extracted IN ORDER. That is, the
// bitstring is first, followed by the att mapped to the lowerest column
// position, followed by the att mapped to the next lowest, and so on.
// The Extract function pulls an attribute out of the hash table...
int lenSoFar = 0, dummy, done;
int lastLen = myEntries[index].Extract (serializeHere, curSlot, hashValue, wayPointID, BITMAP, dummy, done);
// if we cannot find a bitstring, there was no tuple here, and we are done
if (lastLen == 0) {
break;
}
// remember the bitstring
bitstringRHS = (QueryIDSet *) serializeHere;
lenSoFar += lastLen;
// next look for other hashed attributes
<?php
foreach ($jDesc->hash_RHS_attr as $att) {
?>
lastLen = myEntries[index].Extract (serializeHere + lenSoFar, curSlot, hashValue, wayPointID, <?php
echo attSlot($att);
?>
, dummy, done);
// see if we got attribute
if (lastLen > 0) {
Deserialize(serializeHere + lenSoFar, <?php
echo $att;
?>
RHSobj);
//<?php
echo attOptimizedDeserialize($att, $att . "RHSobj", "serializeHere", "lenSoFar");
?>
;
<?php
echo $att;
?>
RHS = &<?php
echo $att;
?>
RHSobj;
lenSoFar += lastLen;
} else {
FATALIF(<?php
echo attQrys($att);
?>
_RHS.Overlaps(*bitstringRHS),
"Did not find attribute <?php
echo $att;
?>
in active RHS tuple");
}
<?php
}
/*foreach*/
?>
// see if we have any query matches
bitstringRHS->Intersect (curBits);
QueryIDSet qBits;
//printf("TPLLLLL: cust_acctbal = %f orders_custkey = %d cust_custkey = %d\n", *customer_c_acctbalRHS, orders_o_custkey_Column.GetCurrent(), *customer_c_custkeyRHS);
<?php
foreach ($jDesc->queries_attribute_comparison as $qClass) {
?>
// See if any query in query class is eligible for this comparision
qBits = QueryIDSet(<?php
echo $qClass->qClass;
?>
, true);
qBits.Intersect(*bitstringRHS);
if (
!qBits.IsEmpty () &&
<?php
foreach ($qClass->att_pairs as $pair) {
?>
*<?php
echo $pair->rhs;
?>
RHS == <?php
echo $pair->lhs;
?>
_Column.GetCurrent() &&
<?php
}
/*foreach pair*/
?>
1 )
{
bitstringLHS.Union (qBits);
}
<?php
}
/*foreach query class*/
?>
// if any of them hit...
if (!bitstringLHS.IsEmpty ()) {
exists.Union(bitstringLHS);
numHits++;
// see if we need to move from shallow to deep
if (numHits == 2 && stillShallow) {
<?php
foreach ($jDesc->attribute_queries_LHS_copy as $att => $qrys) {
?>
<?php
echo attData($att);
?>
_Out.CreateDeepCopy (<?php
echo attData($att);
?>
);
<?php
echo attData($att);
?>
_Out.Insert (<?php
echo attData($att);
?>
.GetCurrent());
<?php
echo attData($att);
?>
_Out.Advance();
<?php
}
/*foreach*/
?>
stillShallow = 0;
}
// now, add all of the outputs over... first deal with the LHS input atts
// that get copied into output atts
if (!stillShallow) {
<?php
foreach ($jDesc->attribute_queries_LHS_copy as $att => $qrys) {
?>
<?php
echo attData($att);
?>
_Out.Insert (<?php
echo attData($att);
?>
.GetCurrent());
<?php
echo attData($att);
?>
_Out.Advance();
<?php
}
/*foreach*/
?>
}
// now, deal with the output atts that come from the hash table
<?php
foreach ($jDesc->attribute_queries_RHS_copy as $att => $qrys) {
?>
<?php
echo attData($att);
?>
_Out.Insert (*<?php
echo $att;
?>
RHS);
<?php
echo attData($att);
?>
_Out.Advance();
<?php
}
/*foreach*/
?>
// finally, set the bitmap. We are one element behind
if (!oldBitstringLHS.IsEmpty()){
myOutBStringIter.Insert (oldBitstringLHS);
myOutBStringIter.Advance ();
}
oldBitstringLHS=bitstringLHS;
} // empty bistring
}
}
// compute the true exist queries
QueryIDSet tmp = existsTarget;
tmp.Intersect(exists);
tmp.Intersect(curBits); // not needed but I'm paranoid
// compute the true not exits queries
QueryIDSet tmp2 = curBits;
tmp2.Intersect(notExistsTarget);
tmp2.Difference(exists);
// now put everything in bitstringLHS
oldBitstringLHS.Union(tmp);
oldBitstringLHS.Union(tmp2);
if (!oldBitstringLHS.IsEmpty()){
myOutBStringIter.Insert (oldBitstringLHS);
myOutBStringIter.Advance ();
}
// at this point, we are done trying to join this tuple... any join results have been
// written to the output columns. Note that we don't have to advance in the output data
// columns; if we are shallow, we don't touch the output columns. If we are not shallow,
// if there were no results, we have nothing to write. HOWEVER, if we are shallow and
// we did not get a match, we need to add an empty btstring
if (stillShallow && numHits == 0) {
<?php
foreach ($jDesc->attribute_queries_RHS_copy as $att => $qrys) {
?>
<?php
echo attType($att);
?>
tmp_<?php
echo attData($att);
?>
;
<?php
echo attData($att);
?>
_Out.Insert (tmp_<?php
echo attData($att);
?>
);
<?php
echo attData($att);
?>
_Out.Advance();
<?php
}
/*foreach*/
?>
if (oldBitstringLHS.IsEmpty()){ // no not exist and no join match
myOutBStringIter.Insert (oldBitstringLHS);
myOutBStringIter.Advance ();
}
}
// lastly, we need to advance in the INPUT tuples
<?php
foreach ($jDesc->attribute_queries_LHS as $att => $qrys) {
?>
<?php
echo attData($att);
?>
.Advance();
<?php
}
/*foreach*/
?>
// advance the input bitstring
myInBStringIter.Advance ();
}
// DONE! So construct the output tuple
// if we are still shallow, put the original data into the output
if (stillShallow) {
<?php
foreach ($jDesc->attribute_queries_LHS_copy as $att => $qrys) {
?>
Column col_<?php
echo $att;
?>
;
<?php
echo attData($att);
?>
.Done(col_<?php
echo $att;
?>
);
output.SwapColumn (col_<?php
echo $att;
?>
, <?php
echo attSlot($att);
?>
);
<?php
}
/*foreach*/
?>
} else {
<?php
foreach ($jDesc->attribute_queries_LHS_copy as $att => $qrys) {
?>
Column col_<?php
echo $att;
?>
;
<?php
echo attData($att);
?>
_Out.Done(col_<?php
echo $att;
?>
);
output.SwapColumn (col_<?php
echo $att;
?>
, <?php
echo attSlot($att);
?>
);
<?php
}
/*foreach*/
?>
}
{
<?php
foreach ($jDesc->attribute_queries_RHS_copy as $att => $qrys) {
?>
Column col_<?php
echo $att;
?>
;
<?php
echo attData($att);
?>
_Out.Done(col_<?php
echo $att;
?>
);
output.SwapColumn (col_<?php
echo $att;
?>
, <?php
echo attSlot($att);
?>
);
<?php
}
/*foreach*/
?>
}
// put in the output bitmap
myOutBStringIter.Done ();
output.SwapBitmap (myOutBStringIter);
// and give back the result
ChunkContainer tempResult (output);
tempResult.swap (result);
PROFILING2_END;
PROFILING(start_time, "<?php
echo $wpName;
?>
", "LHS_lookup", "%d", totalNum);
PROFILING(0.0, "HashTable", "fillrate", "%2.4f", HashTableSegment::globalFillRate*100.0);
// Finish performance counters
// Use the Set functionality in case we add additional counters later.
PCounterList counterList;
PCounter totalCnt("tpi lhs", totalNum, "<?php
echo $wpName;
?>
");
counterList.Append(totalCnt);
PROFILING2_SET(counterList, "<?php
echo $wpName;
?>
");
int64_t hFillRate = int64_t(HashTableSegment::globalFillRate * 1000);
PROFILING2_INSTANT("hfr", hFillRate, "global");
free (serializeHere);
return 1;
}
//+{"kind":"WPF", "name":"LHS Lookup", "action":"end"}
<?php
}
function GLAGenerate_Finalize($wpName, $queries, $attMap)
{
?>
#ifndef PER_QUERY_PROFILE
#define PER_QUERY_PROFILE
#endif
//+{"kind":"WPF", "name":"Finalize (Chunk)", "action":"start"}
extern "C"
int GLAFinalizeWorkFunc_<?php
echo $wpName;
?>
(WorkDescription &workDescription, ExecEngineData &result) {
GLAFinalizeWD myWork;
myWork.swap (workDescription);
QueryExit whichOne = myWork.get_whichQueryExit();
GLAState& glaState = myWork.get_glaState();
<?php
cgDeclareQueryIDs($queries);
?>
// Set up the output chunk
Chunk output;
QueryIDSet queriesToRun = whichOne.query;
<?php
// Extract the state for the query
foreach ($queries as $query => $info) {
$gla = $info['gla'];
?>
// Do query <?php
echo queryName($query);
?>
:
<?php
echo $gla;
?>
* state_<?php
echo queryName($query);
?>
= NULL;
if( whichOne.query == <?php
echo queryName($query);
?>
) {
// Look for the state of query <?php
echo queryName($query);
?>
.
GLAPtr state;
state.swap(glaState);
FATALIF( state.get_glaType() != <?php
echo $gla->cHash();
?>
,
"Got GLA of unexpected type");
state_<?php
echo queryName($query);
?>
= (<?php
echo $gla;
?>
*) state.get_glaPtr();
}
<?php
}
// foreach query
?>
// Start columns for all possible outputs.
<?php
foreach ($queries as $query => $info) {
$output = $info['output'];
cgConstructColumns($output);
}
// foreach query
?>
// This is the output bitstring
MMappedStorage myStore;
Column bitmapOut( myStore );
BStringIterator myOutBStringIter( bitmapOut, queriesToRun );
PROFILING2_START;
int64_t numTuples = 0;
#ifdef PER_QUERY_PROFILE
<?php
foreach ($queries as $query => $info) {
?>
int64_t numTuples_<?php
echo queryName($query);
?>
= 0;
<?php
}
// foreach query
?>
#endif // PER_QUERY_PROFILE
// Extract results
<?php
foreach ($queries as $query => $info) {
$gla = $info['gla'];
$output = $info['output'];
// If this is true, we return the GLA as a const state.
// Otherwise, we pack the results into a chunk.
$retState = $info['retState'];
$stateName = 'state_' . queryName($query);
?>
if( whichOne.query == <?php
echo queryName($query);
?>
) {
<?php
if ($retState) {
?>
FATAL( "Called normal finalize for query that was supposed to be returned as a const state" );
<?php
} else {
$resType = $gla->result_type();
$resType = get_first_value($resType, ['fragment', 'multi', 'single', 'state']);
if ($resType == 'single') {
?>
<?php
echo $stateName;
?>
->GetResult(<?php
echo implode(', ', $output);
?>
);
numTuples++;
#ifdef PER_QUERY_PROFILE
numTuples_<?php
echo queryName($query);
?>
++;
#endif // PER_QUERY_PROFILE
myOutBStringIter.Insert(<?php
echo queryName($query);
?>
);
myOutBStringIter.Advance();
<?php
cgInsertAttributesList($output, '_Column_Out', 2);
} elseif ($resType == 'multi') {
?>
<?php
echo $stateName;
?>
->Finalize();
while( <?php
echo $stateName;
?>
->GetNextResult(<?php
echo implode(', ', $output);
?>
) ) {
numTuples++;
#ifdef PER_QUERY_PROFILE
numTuples_<?php
echo queryName($query);
?>
++;
#endif // PER_QUERY_PROFILE
myOutBStringIter.Insert(<?php
echo queryName($query);
?>
);
myOutBStringIter.Advance();
<?php
cgInsertAttributesList($output, '_Column_Out', 3);
?>
}
<?php
} elseif ($resType == 'fragment') {
?>
int fragment = myWork.get_fragmentNo();
<?php
echo $gla;
?>
_Iterator * iterator = <?php
echo $stateName;
?>
->Finalize(fragment);
while( <?php
echo $stateName;
?>
->GetNextResult(iterator, <?php
echo implode(', ', $output);
?>
) ) {
numTuples++;
#ifdef PER_QUERY_PROFILE
numTuples_<?php
echo queryName($query);
?>
++;
#endif // PER_QUERY_PROFILE
myOutBStringIter.Insert(<?php
echo queryName($query);
?>
);
myOutBStringIter.Advance();
<?php
cgInsertAttributesList($output, '_Column_Out', 3);
?>
}
delete iterator;
<?php
} elseif ($resType == 'state') {
reset($output);
$att = current($output);
// Output attribute
if ($gla->finalize_as_state()) {
?>
<?php
echo $stateName;
?>
->FinalizeState();
<?php
}
// if GLA finalized as state
?>
<?php
echo $att;
?>
= <?php
echo $att->type();
?>
( <?php
echo $stateName;
?>
);
numTuples++;
#ifdef PER_QUERY_PROFILE
numTuples_<?php
echo queryName($query);
?>
++;
#endif // PER_QUERY_PROFILE
myOutBStringIter.Insert(<?php
echo queryName($query);
?>
);
myOutBStringIter.Advance();
<?php
cgInsertAttributesList($output, '_Column_Out', 2);
} else {
grokit_error('GLA ' . $gla . ' has no known result type: [' . implode(',', $resType) . ']');
}
// switch GLA result type
}
// else GLA produces into a chunk
?>
myOutBStringIter.Done();
output.SwapBitmap(myOutBStringIter);
// Write columns
<?php
foreach ($output as $att) {
?>
Column col_<?php
echo $att;
?>
;
<?php
echo $att;
?>
_Column_Out.Done(col_<?php
echo $att;
?>
);
output.SwapColumn( col_<?php
echo $att;
?>
, <?php
echo $att->slot();
?>
);
<?php
}
// foreach output attribute
?>
}
<?php
}
// foreach query
?>
PROFILING2_END;
PCounterList counterList;
PCounter totalCnt("tpo", numTuples, "<?php
echo $wpName;
?>
");
counterList.Append(totalCnt);
#ifdef PER_QUERY_PROFILE
<?php
foreach ($queries as $query => $info) {
?>
{
PCounter qCount("tpo <?php
echo queryName($query);
?>
", numTuples_<?php
echo queryName($query);
?>
, "<?php
echo $wpName;
?>
");
counterList.Append(qCount);
}
<?php
}
// foreach query
?>
#endif // PER_QUERY_PROFILE
PROFILING2_SET(counterList, "<?php
echo $wpName;
?>
");
ChunkContainer tempResult(output);
tempResult.swap(result);
return WP_FINALIZE;
}
//+{"kind":"WPF", "name":"Finalize (Chunk)", "action":"end"}
<?php
}
function GTGenerate_ProcessChunk($wpName, $queries, $attMap)
{
$allPassthrough = [];
$allSynth = [];
foreach ($queries as $query => $info) {
$pass = $info['pass'];
foreach ($pass as $attr) {
if (!in_array($attr, $allPassthrough)) {
$allPassthrough[] = $attr;
}
}
$output = $info['output'];
foreach ($output as $attr) {
if (!in_array($attr, $allSynth)) {
$allSynth[] = $attr;
}
}
}
$gtVars = [];
?>
//+{"kind":"WPF", "name":"Process Chunk", "action":"start"}
extern "C"
int GTProcessChunkWorkFunc_<?php
echo $wpName;
?>
(WorkDescription & workDescription, ExecEngineData & result) {
GTProcessChunkWD myWork;
myWork.swap(workDescription);
Chunk & input = myWork.get_chunkToProcess();
QueryToGLAStateMap & gtStates = myWork.get_gtStates();
QueryToGLAStateMap & constStates = myWork.get_constStates();
QueryIDSet queriesToRun = QueryExitsToQueries( myWork.get_whichQueryExits() );
FATALIF(queriesToRun.IsEmpty(), "Attempted to process chunk with no queries to run.");
<?php
cgDeclareQueryIDs($queries);
?>
QueryIDSet queriesAvailable;
<?php
foreach ($queries as $query => $info) {
?>
queriesAvailable.Union(<?php
echo queryName($query);
?>
);
<?php
}
// for each query
?>
QueryIDSet queriesCovered = queriesToRun;
queriesCovered.Intersect(queriesAvailable);
FATALIF( queriesCovered.IsEmpty(), "Queries being run do not overlap queries known by this waypoint" );
PROFILING2_START;
// Set up the output chunk
Chunk output;
// Defining the GT states needed.
// For each one we will look for an existing state, if we find none, we
// create a new state from scratch.
<?php
foreach ($queries as $query => $info) {
$gt = $info['gt'];
$gtVar = 'state_' . queryName($query);
$gtVars[$query] = $gtVar;
$cstArgs = [];
if ($gt->configurable()) {
echo ' // JSON Configuration for query ' . queryName($query) . PHP_EOL;
$carg = $info['cargs'];
$carg->init();
$cstArgs[] = $carg->name();
echo PHP_EOL;
}
if ($gt->has_state()) {
$cstArgs[] = '*constState';
}
if (\count($cstArgs) > 0) {
$cstStr = '(' . implode(', ', $cstArgs) . ')';
} else {
$cstStr = '';
}
?>
<?php
echo $gt;
?>
* <?php
echo $gtVar;
?>
= nullptr;
if( queriesToRun.Overlaps(<?php
echo queryName($query);
?>
) ) {
if( gtStates.IsThere(<?php
echo queryName($query);
?>
) ) {
// Extract from container
GLAPtr tState;
GLAState& state = gtStates.Find(<?php
echo queryName($query);
?>
);
tState.swap(state);
// Check type
FATALIF( tState.get_glaType() != <?php
echo $gt->cHash();
?>
,
"Got different type than expected for GT of type <?php
echo $gt;
?>
");
// Extract pointer
<?php
echo $gtVar;
?>
= (<?php
echo $gt;
?>
*) tState.get_glaPtr();
// Return container
tState.swap(state);
} // if we already have a state available
else {
// We don't have a state, create a new one
<?php
if ($gt->has_state()) {
?>
// Extract the constant state, so we can use it to initialize
// the GT
const <?php
echo $gt->state();
?>
* constState = nullptr;
GLAPtr constStatePtr;
// Extract from container
FATALIF( !constStates.IsThere(<?php
echo queryName($query);
?>
),
"No constant state found for query (<?php
echo queryName($query);
?>
) that requires a constant state");
GLAState & tempState = constStates.Find(<?php
echo queryName($query);
?>
);
constStatePtr.swap(tempState);
// Check validity
FATALIF( constStatePtr.get_glaType() != <?php
echo $gt->state()->cHash();
?>
,
"Got different type than expected for constant state of type <?php
echo $gt->state();
?>
");
// Get pointer
constState = (const <?php
echo $gt->state();
?>
*) constStatePtr.get_glaPtr();
// Put back in container
constStatePtr.swap(tempState);
<?php
}
// if GT has a constant state
?>
// Construct new GT
<?php
echo $gtVar;
?>
= new <?php
echo $gt;
echo $cstStr;
?>
;
// Play into container, so we can reuse it.
GLAPtr newPtr( <?php
echo $gt->cHash();
?>
, <?php
echo $gtVar;
?>
);
QueryIDSet qry = <?php
echo queryName($query);
?>
;
gtStates.Insert(qry, newPtr);
} // if we had to create a state
} // if query <?php
echo queryName($query);
?>
is running
<?php
}
// foreach query
?>
// Prepare output bitstring iterator
MMappedStorage queries_out_store;
Column queries_out_col( queries_out_store );
BStringIterator queries_out(queries_out_col, queriesCovered);
// Prepare output for all passthrough attributes
<?php
cgConstructColumns($allPassthrough, '_out');
?>
// Prepare output for synthesized attributes
<?php
cgConstructColumns($allSynth, '_out');
?>
// Queries Covered by each passthrough attribute
<?php
foreach ($allPassthrough as $attr) {
$type = $attr->type();
$nullable = $type->is('nullable');
$cstr = "";
if ($nullable) {
$cstr = "(GrokitNull::Value)";
}
?>
<?php
echo $type;
?>
<?php
echo $attr->name();
?>
_out_default<?php
echo $cstr;
?>
;
QueryIDSet <?php
echo $attr->name();
?>
_out_queries;
<?php
foreach ($queries as $query => $info) {
if (in_array($attr, $info['pass'])) {
?>
<?php
echo $attr->name();
?>
_out_queries.Union(<?php
echo queryName($query);
?>
);
<?php
}
// if attribute is part of this query
}
// foreach query
}
// foreach passthrough attribute
?>
// Queries covered by each synthesized attribute
<?php
foreach ($allSynth as $attr) {
$type = $attr->type();
$nullable = $type->is('nullable');
$cstr = "";
if ($nullable) {
$cstr = "(GrokitNull::Value)";
}
?>
<?php
echo $type;
?>
<?php
echo $attr->name();
?>
_out_default<?php
echo $cstr;
?>
;
QueryIDSet <?php
echo $attr->name();
?>
_out_queries;
<?php
foreach ($queries as $query => $info) {
if (in_array($attr, $info['output'])) {
?>
<?php
echo $attr->name();
?>
_out_queries.Union(<?php
echo queryName($query);
?>
);
<?php
}
// if attribute is part of this query
}
// foreach query
}
// foreach passthrough attribute
?>
// Define constants used in expressions
<?php
foreach ($queries as $query => $info) {
$input = $info['expressions'];
cgDeclareConstants($input);
}
// foreach query
?>
// Profiling information
int64_t numTuplesIn = 0;
int64_t numTuplesOut = 0;
#ifdef PER_QUERY_PROFILE
<?php
foreach ($queries as $query => $info) {
?>
int64_t numTuplesIn_<?php
echo queryName($query);
?>
= 0;
int64_t numTuplesOut_<?php
echo queryName($query);
?>
= 0;
<?php
}
// foreach query
?>
#endif // PER_QUERY_PROFILE
// Define should_iterate and is_done for each query
<?php
$iterVars = [];
foreach ($queries as $query => $info) {
$iterVar = 'should_process_' . queryName($query);
$iterVars[] = $iterVar;
?>
bool <?php
echo $iterVar;
?>
= true;
<?php
}
$iterExpr = implode(' || ', $iterVars);
?>
<?php
foreach ($queries as $query => $info) {
// Call StartChunk() on iterable GTs
$gt = $info['gt'];
$gtVar = $gtVars[$query];
if ($gt->iterable()) {
?>
if (queriesToRun.Overlaps(<?php
echo queryName($query);
?>
)) {
<?php
echo $gtVar;
?>
->StartChunk();
}
<?php
}
}
?>
do {
<?php
cgAccessColumns($attMap, 'input', $wpName);
?>
// Prepare input bitstring iterator
BStringIterator queries_in;
input.SwapBitmap(queries_in);
numTuplesIn = 0;
#ifdef PER_QUERY_PROFILE
<?php
foreach ($queries as $query => $info) {
?>
numTuplesIn_<?php
echo queryName($query);
?>
= 0;
<?php
}
// foreach query
?>
#endif // PER_QUERY_PROFILE
// Begin processing tuples
while( !queries_in.AtEndOfColumn() ) {
numTuplesIn++;
QueryIDSet qry = queries_in.GetCurrent();
qry.Intersect(queriesToRun);
queries_in.Advance();
<?php
cgAccessAttributes($attMap);
foreach ($queries as $query => $info) {
$gt = $info['gt'];
$input = $info['expressions'];
$output = $info['output'];
$iterVar = 'should_process_' . queryName($query);
$gtVar = $gtVars[$query];
$outputVars = [];
foreach ($output as $attr) {
$outputVars[] = $attr->name() . '_out';
}
$inputVals = [];
foreach ($input as $expr) {
$inputVals[] = $expr->value();
}
// Prepare the output text
ob_start();
?>
// Insert synthesized attributes
<?php
foreach ($allSynth as $attr) {
?>
if( queriesToRun.Overlaps(<?php
echo $attr->name();
?>
_out_queries) ) {
<?php
echo $attr->name();
?>
_out_Column_Out.Insert(<?php
echo $attr->name();
?>
_out);
}
else {
<?php
echo $attr->name();
?>
_out_Column_Out.Insert(<?php
echo $attr->name();
?>
_out_default);
}
<?php
echo $attr->name();
?>
_out_Column_Out.Advance();
<?php
}
// foreach synthesized attribute
?>
// Insert passthrough attributes
<?php
foreach ($allPassthrough as $attr) {
?>
if( queriesToRun.Overlaps(<?php
echo $attr->name();
?>
_out_queries) ) {
<?php
echo $attr->name();
?>
_out_Column_Out.Insert(<?php
echo $attr->name();
?>
);
}
else {
<?php
echo $attr->name();
?>
_out_Column_Out.Insert(<?php
echo $attr->name();
?>
_out_default);
}
<?php
echo $attr->name();
?>
_out_Column_Out.Advance();
<?php
}
// foreach synthesized attribute
?>
queries_out.Insert(<?php
echo queryName($query);
?>
);
queries_out.Advance();
numTuplesOut++;
#ifdef PER_QUERY_PROFILE
numTuplesOut_<?php
echo queryName($query);
?>
++;
#endif // PER_QUERY_PROFILE
<?php
$outputText = ob_get_clean();
?>
// Do Query [<?php
echo queryName($query);
?>
]
if( <?php
echo $iterVar;
?>
&& qry.Overlaps(<?php
echo queryName($query);
?>
) ) {
#ifdef PER_QUERY_PROFILE
numTuplesIn_<?php
echo queryName($query);
?>
++;
#endif // PER_QUERY_PROFILE
<?php
cgDeclarePreprocessing($input, 3);
?>
<?php
if ($gt->result_type() == 'single') {
?>
if( <?php
echo $gtVar;
?>
->ProcessTuple(<?php
echo implode(', ', $inputVals);
?>
, <?php
echo implode(', ', $outputVars);
?>
) ) {
<?php
echo $outputText;
?>
} // if GT produced output for this tuple
<?php
} else {
// if gt result type is single
?>
<?php
echo $gtVar;
?>
->ProcessTuple(<?php
echo implode(', ', $inputVals);
?>
);
while( <?php
echo $gtVar;
?>
->GetNextResult(<?php
echo implode(', ', $outputVars);
?>
) ) {
<?php
echo $outputText;
?>
} // while GT has output for this tuple
<?php
}
// if gt result type is multi
?>
} // if <?php
echo queryName($query);
?>
active in tuple
<?php
}
// foreach query
cgAdvanceAttributes($attMap, 2);
?>
} // while we have tuples to process
<?php
foreach ($queries as $query => $info) {
$gt = $info['gt'];
$iterVar = 'should_process_' . queryName($query);
$gtVar = $gtVars[$query];
if ($gt->iterable()) {
?>
if (<?php
echo $iterVar;
?>
) {
<?php
echo $iterVar;
?>
= <?php
echo $gtVar;
?>
->ShouldIterate();
}
<?php
} else {
?>
<?php
echo $iterVar;
?>
= false;
<?php
}
}
//foreach query
?>
// Put columns back into chunk
<?php
cgPutbackColumns($attMap, 'input', $wpName);
?>
// Put bitmap back into chunk
queries_in.Done();
input.SwapBitmap(queries_in);
} while(<?php
echo $iterExpr;
?>
);
// Tell GTs that need to know about the chunk boundary
<?php
foreach ($queries as $query => $info) {
$gt = $info['gt'];
$gtVar = $gtVars[$query];
if ($gt->chunk_boundary()) {
?>
<?php
echo $gtVar;
?>
->ChunkBoundary();
<?php
}
// if GT cares about chunk boundaries
}
// foreach query
?>
// Finalize the output iterators and put the columns into the output chunk
queries_out.Done();
output.SwapBitmap(queries_out);
<?php
foreach ($allPassthrough as $attr) {
?>
if( queriesToRun.Overlaps(<?php
echo $attr->name();
?>
_out_queries) ) {
Column temp;
<?php
echo $attr->name();
?>
_out_Column_Out.Done(temp);
output.SwapColumn(temp, <?php
echo $attr->slot();
?>
);
}
<?php
}
// foreach passthrough attribute
?>
<?php
foreach ($allSynth as $attr) {
?>
if( queriesToRun.Overlaps(<?php
echo $attr->name();
?>
_out_queries) ) {
Column temp;
<?php
echo $attr->name();
?>
_out_Column_Out.Done(temp);
output.SwapColumn(temp, <?php
echo $attr->slot();
?>
);
}
<?php
}
// foreach passthrough attribute
?>
PROFILING2_END;
// Profiling
PCounterList counterList;
PCounter totalCntIn("tpi", numTuplesIn, "<?php
echo $wpName;
?>
");
counterList.Append(totalCntIn);
PCounter totalCntOut("tpo", numTuplesOut, "<?php
echo $wpName;
?>
");
counterList.Append(totalCntOut);
#ifdef PER_QUERY_PROFILE
<?php
foreach ($queries as $query => $info) {
?>
if( queriesToRun.Overlaps(<?php
echo queryName($query);
?>
) ) {
PCounter cntIn("tpi <?php
echo queryName($query);
?>
", numTuplesIn_<?php
echo queryName($query);
?>
, "<?php
echo $wpName;
?>
");
counterList.Append(cntIn);
PCounter cntOut("tpo <?php
echo queryName($query);
?>
", numTuplesOut_<?php
echo queryName($query);
?>
, "<?php
echo $wpName;
?>
");
counterList.Append(cntOut);
}
<?php
}
// foreach query
?>
#endif // PER_QUERY_PROFILE
PROFILING2_SET(counterList, "<?php
echo $wpName;
?>
");
// Return result and exit
GTProcessChunkRez gtRez(gtStates, output);
result.swap(gtRez);
return WP_PROCESS_CHUNK;
}
//+{"kind":"WPF", "name":"Process Chunk", "action":"end"}
<?php
}
function GIGenerate($wpName, $type, $outputs, $constArgs, $tuples)
{
// put an element in front to form constant arguments
array_unshift($constArgs, "stream_info");
$constArgsStr = join($constArgs, ',');
// comma separated list of outputs
$outputArgs = join(array_map(function ($el) {
return $el->name();
}, $outputs), ',');
?>
#include "Dictionary.h"
#include "DictionaryManager.h"
#include "GIStreamInfo.h"
#include "Profiling.h"
#include <iostream>
//+{"kind":"WPF", "name":"Produce Chunk", "action":"start"}
extern "C"
int GIProduceChunkWorkFunc_<?php
echo $wpName;
?>
( WorkDescription &workDescription, ExecEngineData &result) {
const size_t maxTuplesPerChunk = <?php
echo $tuples == 0 ? "PREFERED_TUPLES_PER_CHUNK" : $tuples;
?>
;
GIProduceChunkWD myWork;
myWork.swap( workDescription );
GLAState &gi_state = myWork.get_gi();
GIStreamProxy &stream_info = myWork.get_stream_info();
QueryIDSet queriesToRun = myWork.get_queriesCovered();
GLAPtr state_ptr;
state_ptr.swap(gi_state);
<?php
echo $type->value();
?>
* my_state = NULL;
if( state_ptr.IsValid() ) {
my_state = (<?php
echo $type->value();
?>
*) state_ptr.get_glaPtr();
} else {
my_state = new <?php
echo $type->value();
?>
( <?php
echo $constArgsStr;
?>
);
GLAPtr newPtr(<?php
echo $type->cHash();
?>
, (void*) my_state);
newPtr.swap(state_ptr);
}
state_ptr.swap(gi_state);
// Output chunk
Chunk chunk;
PROFILING2_START;
// Start new columns and allocate storage for them
<?php
cgConstructColumns($outputs);
?>
size_t tuple_count = 0;
bool stream_done = false;
while( !stream_done && tuple_count < maxTuplesPerChunk ) {
if( __builtin_expect( my_state->ProduceTuple( <?php
echo $outputArgs;
?>
), 1) ) {
++tuple_count;
<?php
foreach ($outputs as $att) {
?>
<?php
echo $att->name();
?>
_Column_Out.Insert(<?php
echo $att->name();
?>
);
<?php
echo $att->name();
?>
_Column_Out.Advance();
<?php
}
?>
}
else {
stream_done = true;
}
}
// Deal with PreDone() calls to release read locks
<?php
foreach ($outputs as $att) {
if ($att->type()->reqDictionary()) {
?>
<?php
echo $att->name();
?>
_Column_Out.PreDone();
<?php
}
}
?>
// Finalize output columns and put them into the chunk.
<?php
foreach ($outputs as $att) {
if ($att->type()->reqDictionary()) {
?>
<?php
echo $att->name();
?>
_Column_Out.Done(<?php
echo $att->name();
?>
_Column_Ocol,
my_state->get_dictionary_<?php
echo $att->type()->dictionary();
?>
());
<?php
} else {
?>
<?php
echo $att->name();
?>
_Column_Out.Done(<?php
echo $att->name();
?>
_Column_Ocol);
<?php
}
?>
{
int cSlot = <?php
echo $att->slot();
?>
;
FATALIF( ! <?php
echo $att->name();
?>
_Column_Ocol.IsValid(), "Column _A_ invalid when packing into chunk!");
chunk.SwapColumn( <?php
echo $att->name();
?>
_Column_Ocol, cSlot );
}
<?php
}
?>
PROFILING2_END;
PROFILING2_SINGLE("tpo", tuple_count, "<?php
echo $wpName;
?>
");
MMappedStorage bitStore;
Column outBitCol(bitStore);
BStringIterator outQueries( outBitCol, queriesToRun, tuple_count );
outQueries.Done();
chunk.SwapBitmap(outQueries);
ChunkContainer chunkCont(chunk);
// pack the chunk, stream, and GI into the result and send it back
GIProduceChunkRez tempResult( stream_info, gi_state, chunkCont );
result.swap(tempResult);
if( stream_done )
{
// Deallocate GI
delete my_state;
GLAState blankState;
blankState.swap(gi_state);
return 1;
}
else{
return 0;
}
}
//+{"kind":"WPF", "name":"Produce Chunk", "action":"end"}
<?php
}
function GISTGenerate_ProduceResults($wpName, $queries, $attMap)
{
?>
extern "C"
int GISTProduceResultsWorkFunc_<?php
echo $wpName;
?>
(WorkDescription &workDescription, ExecEngineData &result) {
GISTProduceResultsWD myWork;
myWork.swap(workDescription);
// Inputs
QueryExit& whichOne = myWork.get_whichOne();
GLAState& gist = myWork.get_gist();
int fragmentNo = myWork.get_fragmentNo();
// Outputs
Chunk output;
<?php
cgDeclareQueryIDs($queries);
?>
QueryIDSet queriesToRun = whichOne.query;
// Start columns for outputs
<?php
foreach ($queries as $query => $info) {
$output = $info['output'];
cgConstructColumns($output);
}
// foreach query
?>
// Output bitstring
MMappedStorage myStore;
Column bitmapOut(myStore);
BStringIterator myOutBStringIter (bitmapOut, queriesToRun);
PROFILING2_START;
int64_t numTuples = 0;
#ifdef PER_QUERY_PROFILE
<?php
foreach ($queries as $query => $info) {
?>
int64_t numTuples_<?php
echo queryName($query);
?>
= 0;
<?php
}
// foreach query
?>
#endif // PER_QUERY_PROFILE
<?php
foreach ($queries as $query => $info) {
$gist = $info['gist'];
$output = $info['output'];
$resType = $gist->result_type();
$resType = get_first_value($resType, ['fragment', 'multi', 'single', 'state']);
?>
if( whichOne.query == <?php
echo queryName($query);
?>
) {
// Extract the GIST state
GLAPtr gistPtr;
gistPtr.swap(gist);
FATALIF(gistPtr.get_glaType() != <?php
echo $gist->cHash();
?>
,
"GIST producing results is of incorrect type for query <?php
echo queryName($query);
?>
");
<?php
echo $gist;
?>
* state_<?php
echo queryName($query);
?>
= (<?php
echo $gist;
?>
*) gistPtr.get_glaPtr();
<?php
switch ($resType) {
case 'single':
?>
{
state_<?php
echo queryName($query);
?>
->GetResult(<?php
echo implode(', ', $output);
?>
);
<?php
break;
case 'multi':
?>
state_<?php
echo queryName($query);
?>
->Finalize();
while (state_<?php
echo queryName($query);
?>
->GetNextResult(<?php
echo implode(', ', $output);
?>
)) {
<?php
break;
case 'fragment':
?>
<?php
echo $gist;
?>
::Iterator* iterator
= state_<?php
echo queryName($query);
?>
->Finalize( fragmentNo );
while( state_<?php
echo queryName($query);
?>
->GetNextResult( iterator, <?php
echo implode(', ', $output);
?>
) ) {
<?php
break;
case 'state':
reset($output);
$att = current($output);
// Output attribute
?>
{
<?php
if ($gist->finalize_as_state()) {
?>
state_<?php
echo queryName($query);
?>
->FinalizeState();
<?php
}
?>
<?php
echo $att;
?>
= <?php
echo $att->type();
?>
( state_<?php
echo queryName($query);
?>
);
<?php
break;
default:
grokit_error("Do not know how to deal with output type of GLA {$gist}::cGla [{$resType}]");
}
// matches switch
?>
numTuples++;
#ifdef PER_QUERY_PROFILE
numTuples_<?php
echo queryName($query);
?>
++;
#endif // PER_QUERY_PROFILE
// Advance the columns
myOutBStringIter.Insert(<?php
echo queryName($query);
?>
);
myOutBStringIter.Advance();
<?php
foreach ($output as $att) {
?>
<?php
echo $att;
?>
_Column_Out.Insert(<?php
echo $att;
?>
);
<?php
echo $att;
?>
_Column_Out.Advance();
<?php
}
?>
} // Matches block for column stuff.
<?php
if ($resType == 'fragment') {
?>
// Delete the iterator;
delete iterator;
<?php
}
?>
} // Matches whichOne
<?php
}
// matches foreach query
?>
myOutBStringIter.Done();
output.SwapBitmap(myOutBStringIter);
// Write columns
<?php
foreach ($queries as $query => $info) {
$gist = $info['gist'];
$output = $info['output'];
?>
if( whichOne.query == <?php
echo queryName($query);
?>
) {
<?php
foreach ($output as $att) {
?>
Column col_<?php
echo $att;
?>
;
<?php
echo $att;
?>
_Column_Out.Done(col_<?php
echo $att;
?>
);
output.SwapColumn(col_<?php
echo $att;
?>
, <?php
echo $att->slot();
?>
);
<?php
}
// foreach output att
?>
} // Matches whichOne
<?php
}
// foreach query
?>
ChunkContainer tempResult(output);
tempResult.swap(result);
return WP_FINALIZE; // ProduceResults
}
<?php
}
