Beispiel #1
0
function JoinLHSHash($wpName, $jDesc)
{
    ?>

//+{"kind":"WPF", "name":"LHS Hash", "action":"start"}
extern "C"
int JoinLHSHashWorkFunc_<?php 
    echo $wpName;
    ?>
 (WorkDescription &workDescription, ExecEngineData &result) {
    double start_time = global_clock.GetTime();

    // this is the area where all of the intermediate, serialized records are stored
    SerializedSegmentArray serializedSegments [NUM_SEGS];

    // this is the area where all of the records are serialized to;
    // 10K bytes are initially used for this
    void *serializeHere = (void *) malloc (10000);
    int storageSize = 10000;

    // go to the work description and get the input chunk
    JoinLHSHashWorkDescription myWork;
    myWork.swap (workDescription);
    Chunk &input = myWork.get_chunkToProcess ();

    // get the waypoint identifier
    unsigned int wayPointID = myWork.get_wayPointID ();

    QueryIDSet queriesToRun = QueryExitsToQueries(myWork.get_whichQueryExits ());

<?php 
    cgAccessColumns($jDesc->attribute_queries_LHS, 'input', $wpName);
    ?>

    Column inBitCol;
    BStringIterator queries;
    input.SwapBitmap (queries);

    int totalNum = 0; // counter for the tuples processed

    // now actually hash all of the tuples!
    while (!queries.AtEndOfColumn ()) {
        QueryIDSet qry;
        qry = queries.GetCurrent();
        qry.Intersect(queriesToRun);
        queries.Advance();

        // extract values of attributes from streams
<?php 
    cgAccessAttributes($jDesc->attribute_queries_LHS);
    ?>

         if (qry.IsEmpty()){
<?php 
    cgAdvanceAttributes($jDesc->attribute_queries_LHS);
    ?>
             continue;
         }

        totalNum++;

        HT_INDEX_TYPE hashValue = HASH_INIT;
<?php 
    foreach ($jDesc->LHS_hash as $att) {
        ?>
        hashValue = CongruentHash(Hash(<?php 
        echo $att;
        ?>
), hashValue);
<?php 
    }
    ?>

        // figure out which of the hash buckets it goes into
        unsigned int index = WHICH_SEGMENT (hashValue);

        // and serialize the record!  Begin with the bitstring.
        int bytesUsed = sizeof(Bitstring);

        // Make sure we have the storage...
        if (bytesUsed > storageSize) {
            storageSize = bytesUsed;
            free (serializeHere);
            serializeHere = (void *) malloc (storageSize);
        }

        // do the serialization...
        void *location = (void*)&qry;

        // remember the serialized value
        serializedSegments[index].StartNew (WHICH_SLOT (hashValue), wayPointID, 0, location, bytesUsed);

        // now, go thru all of the attributes that are used
<?php 
    foreach ($jDesc->LHS_hash as $att) {
        ?>
        bytesUsed = <?php 
        echo attSerializedSize($att, $att);
        ?>
;
        if (bytesUsed > storageSize) {
            storageSize = bytesUsed;
            free (serializeHere);
            serializeHere = (void *) malloc (storageSize);
        }

        // and record the serialized value
        location =  <?php 
        echo attOptimizedSerialize($att, $att, "serializeHere");
        ?>
;
        serializedSegments[index].Append (<?php 
        echo attSlot($att);
        ?>
, location, bytesUsed);
<?php 
    }
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $query) {
        ?>
        if (qry.Overlaps(QueryIDSet(<?php 
        echo queryName($query);
        ?>
, true))){

            bytesUsed = <?php 
        echo attSerializedSize($att, $att);
        ?>
;
            if (bytesUsed > storageSize) {
                storageSize = bytesUsed;
                free (serializeHere);
                serializeHere = (void *) malloc (storageSize);
            }

            // and record the serialized value
            location =  <?php 
        echo attOptimizedSerialize($att, $att, "serializeHere");
        ?>
;
            serializedSegments[index].Append (<?php 
        echo attSlot($att);
        ?>
, location, bytesUsed);
        }
<?php 
    }
    ?>

<?php 
    cgAdvanceAttributes($jDesc->attribute_queries_LHS);
    ?>

    } // for each tuple

    // now we are done serializing the chunk
    free (serializeHere);

    // so actually do the hashing... first set up the list of the guys we want to hash
    int theseAreOK [NUM_SEGS];
    for (int i = 0; i < NUM_SEGS; i++) {
        theseAreOK[i] = 1;
    }

    // this is the set of sample collisions taken from the over-full segments
    HashSegmentSample mySamples;

    // now go through and, one-at-a-time, add the data to each table segment
    for (int i = 0; i < NUM_SEGS; i++) {

        // first get a segment to add data to
        HashTableSegment checkedOutCopy;
        int whichOne = myWork.get_centralHashTable ().CheckOutOne (theseAreOK, checkedOutCopy);
        theseAreOK[whichOne] = 0;

        // now add the data
        HashSegmentSample mySample;
        if (checkedOutCopy.Insert (serializedSegments[whichOne], mySample)) {

            // if we are in here, it means that the segment was over-full, so note that we will
            // need to empty it out... we record all of the samples
            mySamples.MoveToFinish ();
            mySample.MoveToStart ();
            mySamples.SwapRights (mySample);
        }

        // and then put the segment back in the hash table
        myWork.get_centralHashTable ().CheckIn (whichOne);
    }

<?php 
    cgPutbackColumns($jDesc->attribute_queries_LHS, 'input', $wpName);
    ?>

    PROFILING(start_time, "<?php 
    echo $wpName;
    ?>
", "LHS_hash", "%d", totalNum);
    PROFILING(0.0, "HashTable", "fillrate", "%2.4f", HashTableSegment::globalFillRate.load());

    // now we are finally done!
    JoinHashResult myResult (mySamples);
    myResult.swap (result);
    return 0;
}
//+{"kind":"WPF", "name":"LHS Hash", "action":"end"}

<?php 
}
Beispiel #2
0
function GLAGenerate_ProcessChunk($wpName, $queries, $attMap)
{
    ?>
#ifndef PER_QUERY_PROFILE
#define PER_QUERY_PROFILE
#endif

#include <map>
#include <sstream>

//+{"kind":"WPF", "name":"Process Chunk", "action":"start"}
extern "C"
int GLAProcessChunkWorkFunc_<?php 
    echo $wpName;
    ?>
(WorkDescription &workDescription, ExecEngineData &result) {

    GLAProcessChunkWD myWork;
    myWork.swap(workDescription);
    Chunk &input = myWork.get_chunkToProcess ();

    QueryToGLAStateMap& glaStates = myWork.get_glaStates();
    QueryToGLAStateMap& constStates = myWork.get_constStates();
    QueryToGLAStateMap& garbageStates = myWork.get_garbageStates();

    PROFILING2_START;

    QueryIDSet queriesToRun = QueryExitsToQueries( myWork.get_whichQueryExits() );
    FATALIF(queriesToRun.IsEmpty(), "Why are we processing a chunk with no queries to run?");

<?php 
    cgDeclareQueryIDs($queries);
    ?>

    QueryIDSet queriesAvailable;
<?php 
    foreach ($queries as $query => $info) {
        ?>
    queriesAvailable.Union(<?php 
        echo queryName($query);
        ?>
);
<?php 
    }
    // for each query
    ?>

    QueryIDSet queriesCovered = queriesToRun;
    queriesCovered.Intersect(queriesAvailable);

    FATALIF( queriesCovered.IsEmpty(), "Queries being run do not overlap queries known by this waypoint!" );

<?php 
    cgAccessColumns($attMap, 'input', $wpName);
    ?>

    // prepare bitstring iterator
    BStringIterator queries;
    input.SwapBitmap (queries);

    // Garbage collect old states, if there are any.
<?php 
    foreach ($queries as $query => $info) {
        $gla = $info['gla'];
        ?>
    if( garbageStates.IsThere( <?php 
        echo queryName($query);
        ?>
 ) ) {
        QueryID curID = <?php 
        echo queryName($query);
        ?>
;
        QueryID key;
        GLAState state;
        garbageStates.Remove( curID, key, state );

        GLAPtr curPtr;
        curPtr.swap(state);

        <?php 
        echo $gla;
        ?>
 * garbage = (<?php 
        echo $gla;
        ?>
 *) curPtr.get_glaPtr();
        delete garbage;
    }
<?php 
    }
    // foreach query
    ?>

    // Defining the GLA states needed.
    // For each one we will look for an existing state, if we find none, we
    // create a state from scratch.
<?php 
    $glaVars = [];
    foreach ($queries as $query => $info) {
        $gla = $info['gla'];
        $glaVar = 'state_' . queryName($query);
        $glaVars[$query] = $glaVar;
        // Create the pointer to the GLA
        $cstArgs = [];
        if ($gla->configurable()) {
            echo '    // JSON Configuration for query ' . queryName($query) . PHP_EOL;
            $carg = $info['cargs'];
            $carg->init();
            $cstArgs[] = $carg->name();
            echo PHP_EOL;
        }
        if ($gla->has_state()) {
            $cstArgs[] = '*constState';
        }
        if (\count($cstArgs) > 0) {
            $cstStr = '(' . implode(', ', $cstArgs) . ')';
        } else {
            $cstStr = '';
        }
        ?>
    <?php 
        echo $gla;
        ?>
 * <?php 
        echo $glaVar;
        ?>
 = NULL;
        if( queriesToRun.Overlaps(<?php 
        echo queryName($query);
        ?>
) ) {
            if( glaStates.IsThere(<?php 
        echo queryName($query);
        ?>
) ) {
                // We already have a state, just extract it.

                GLAPtr tState;
                GLAState& state = glaStates.Find(<?php 
        echo queryName($query);
        ?>
);
                tState.swap(state);
                FATALIF( tState.get_glaType() != <?php 
        echo $gla->cHash();
        ?>
,
                    "Got different type than expected for GLA of type <?php 
        echo $gla;
        ?>
");
                <?php 
        echo $glaVar;
        ?>
 = (<?php 
        echo $gla;
        ?>
 *) tState.get_glaPtr();
                tState.swap(state);
            }
            else {
                // We don't have a state, create a new one.
<?php 
        if ($gla->has_state()) {
            ?>
                // Extract the constant state, so we can use it to initialize
                // the GLA
                const <?php 
            echo $gla->state();
            ?>
 * constState = nullptr;
                GLAPtr constStatePtr;

                // Extract from container
                FATALIF( !constStates.IsThere(<?php 
            echo queryName($query);
            ?>
),
                    "No constant state found for query (<?php 
            echo queryName($query);
            ?>
) that requires a constant state");
                GLAState & tempState = constStates.Find(<?php 
            echo queryName($query);
            ?>
);
                constStatePtr.swap(tempState);

                // Check validity
                FATALIF( constStatePtr.get_glaType() != <?php 
            echo $gla->state()->cHash();
            ?>
,
                    "Got different type than expected for constant state of type <?php 
            echo $gla->state();
            ?>
");

                // Get pointer
                constState = (const <?php 
            echo $gla->state();
            ?>
 *) constStatePtr.get_glaPtr();

                // Put back in container
                constStatePtr.swap(tempState);

<?php 
        }
        // if gla has a constant state
        ?>
                // Construct new GLA
                <?php 
        echo $glaVar;
        ?>
 = new <?php 
        echo $gla;
        echo $cstStr;
        ?>
;

                // Place into container, so we can reuse it.
                GLAPtr newPtr( <?php 
        echo $gla->cHash();
        ?>
, <?php 
        echo $glaVar;
        ?>
 );
                QueryIDSet qry = <?php 
        echo queryName($query);
        ?>
;
                glaStates.Insert(qry, newPtr);
            } // else don't have a GLA state already
    } // if queriesToRun overlaps <?php 
        echo queryName($query);
        ?>
.
<?php 
    }
    // foreach query
    // Define constants used in expressions
    foreach ($queries as $query => $info) {
        $input = $info['expressions'];
        cgDeclareConstants($input);
        // Per-query profile
        ?>
#ifdef PER_QUERY_PROFILE
    int64_t numTuples_<?php 
        echo queryName($query);
        ?>
 = 0;
#endif // PER_QUERY_PROFILE
<?php 
    }
    // foreach query
    ?>
    int64_t numTuples = 0;
    while( !queries.AtEndOfColumn() ) {
        ++numTuples;
        QueryIDSet qry;
        qry = queries.GetCurrent();
        qry.Intersect(queriesToRun);
        queries.Advance();

<?php 
    cgAccessAttributes($attMap);
    foreach ($queries as $query => $info) {
        $gla = $info['gla'];
        $input = $info['expressions'];
        $output = $info['output'];
        $glaVar = $glaVars[$query];
        ?>
        // Do query <?php 
        echo queryName($query);
        ?>
:
        if( qry.Overlaps(<?php 
        echo queryName($query);
        ?>
) ) {
<?php 
        // Declare preprocessing variables
        cgDeclarePreprocessing($input, 3);
        ?>
            <?php 
        echo $glaVar;
        ?>
->AddItem( <?php 
        echo implode(', ', $input);
        ?>
);

#ifdef PER_QUERY_PROFILE
            numTuples_<?php 
        echo queryName($query);
        ?>
++;
#endif // PER_QUERY_PROFILE
        } // if query overlaps <?php 
        echo queryName($query);
        ?>
.
<?php 
    }
    // foreach query
    cgAdvanceAttributes($attMap, 2);
    ?>
    } // while not at end of input

<?php 
    // Tell GLAs that wish to know about the chunk boundary.
    foreach ($queries as $query => $info) {
        $gla = $info['gla'];
        $glaVar = $glaVars[$query];
        if ($gla->chunk_boundary()) {
            ?>
    <?php 
            echo $glaVar;
            ?>
->ChunkBoundary();
<?php 
        }
        // if GLA wishes to know about chunk boundary
    }
    // foreach query
    ?>
    PROFILING2_END;

    PCounterList counterList;
    PCounter totalCnt("tpi", numTuples, "<?php 
    echo $wpName;
    ?>
");
    counterList.Append(totalCnt);

#ifdef PER_QUERY_PROFILE
    // Add tuple counters to list
<?php 
    foreach ($queries as $query => $info) {
        ?>
    PCounter cnt("tpi <?php 
        echo queryName($query);
        ?>
", numTuples_<?php 
        echo queryName($query);
        ?>
, "<?php 
        echo $wpName;
        ?>
");
    counterList.Append(cnt);
<?php 
    }
    //foreach query
    ?>
#endif // PER_QUERY_PROFILE

    PROFILING2_SET(counterList, "<?php 
    echo $wpName;
    ?>
");

<?php 
    cgPutbackColumns($attMap, 'input', $wpName);
    ?>
    // Finish bitstring iterator
    queries.Done();
    input.SwapBitmap(queries);

    GLAProcessRez glaResult(glaStates, input);
    result.swap(glaResult);

    return WP_PROCESS_CHUNK; // for processchunk
}
//+{"kind":"WPF", "name":"Process Chunk", "action":"end"}
<?php 
}
Beispiel #3
0
function GTGenerate_ProcessChunk($wpName, $queries, $attMap)
{
    $allPassthrough = [];
    $allSynth = [];
    foreach ($queries as $query => $info) {
        $pass = $info['pass'];
        foreach ($pass as $attr) {
            if (!in_array($attr, $allPassthrough)) {
                $allPassthrough[] = $attr;
            }
        }
        $output = $info['output'];
        foreach ($output as $attr) {
            if (!in_array($attr, $allSynth)) {
                $allSynth[] = $attr;
            }
        }
    }
    $gtVars = [];
    ?>
//+{"kind":"WPF", "name":"Process Chunk", "action":"start"}
extern "C"
int GTProcessChunkWorkFunc_<?php 
    echo $wpName;
    ?>
(WorkDescription & workDescription, ExecEngineData & result) {
    GTProcessChunkWD myWork;
    myWork.swap(workDescription);

    Chunk & input = myWork.get_chunkToProcess();

    QueryToGLAStateMap & gtStates = myWork.get_gtStates();
    QueryToGLAStateMap & constStates = myWork.get_constStates();

    QueryIDSet queriesToRun = QueryExitsToQueries( myWork.get_whichQueryExits() );
    FATALIF(queriesToRun.IsEmpty(), "Attempted to process chunk with no queries to run.");

<?php 
    cgDeclareQueryIDs($queries);
    ?>

    QueryIDSet queriesAvailable;
<?php 
    foreach ($queries as $query => $info) {
        ?>
    queriesAvailable.Union(<?php 
        echo queryName($query);
        ?>
);
<?php 
    }
    // for each query
    ?>

    QueryIDSet queriesCovered = queriesToRun;
    queriesCovered.Intersect(queriesAvailable);

    FATALIF( queriesCovered.IsEmpty(), "Queries being run do not overlap queries known by this waypoint" );

    PROFILING2_START;

    // Set up the output chunk
    Chunk output;

    // Defining the GT states needed.
    // For each one we will look for an existing state, if we find none, we
    // create a new state from scratch.

<?php 
    foreach ($queries as $query => $info) {
        $gt = $info['gt'];
        $gtVar = 'state_' . queryName($query);
        $gtVars[$query] = $gtVar;
        $cstArgs = [];
        if ($gt->configurable()) {
            echo '    // JSON Configuration for query ' . queryName($query) . PHP_EOL;
            $carg = $info['cargs'];
            $carg->init();
            $cstArgs[] = $carg->name();
            echo PHP_EOL;
        }
        if ($gt->has_state()) {
            $cstArgs[] = '*constState';
        }
        if (\count($cstArgs) > 0) {
            $cstStr = '(' . implode(', ', $cstArgs) . ')';
        } else {
            $cstStr = '';
        }
        ?>
    <?php 
        echo $gt;
        ?>
 * <?php 
        echo $gtVar;
        ?>
 = nullptr;
    if( queriesToRun.Overlaps(<?php 
        echo queryName($query);
        ?>
) ) {
        if( gtStates.IsThere(<?php 
        echo queryName($query);
        ?>
) ) {
            // Extract from container
            GLAPtr tState;
            GLAState& state = gtStates.Find(<?php 
        echo queryName($query);
        ?>
);
            tState.swap(state);

            // Check type
            FATALIF( tState.get_glaType() != <?php 
        echo $gt->cHash();
        ?>
,
                "Got different type than expected for GT of type <?php 
        echo $gt;
        ?>
");

            // Extract pointer
            <?php 
        echo $gtVar;
        ?>
 = (<?php 
        echo $gt;
        ?>
 *) tState.get_glaPtr();

            // Return container
            tState.swap(state);
        } // if we already have a state available
        else {
            // We don't have a state, create a new one
<?php 
        if ($gt->has_state()) {
            ?>

            // Extract the constant state, so we can use it to initialize
            // the GT
            const <?php 
            echo $gt->state();
            ?>
 * constState = nullptr;
            GLAPtr constStatePtr;

            // Extract from container
            FATALIF( !constStates.IsThere(<?php 
            echo queryName($query);
            ?>
),
                "No constant state found for query (<?php 
            echo queryName($query);
            ?>
) that requires a constant state");
            GLAState & tempState = constStates.Find(<?php 
            echo queryName($query);
            ?>
);
            constStatePtr.swap(tempState);

            // Check validity
            FATALIF( constStatePtr.get_glaType() != <?php 
            echo $gt->state()->cHash();
            ?>
,
                "Got different type than expected for constant state of type <?php 
            echo $gt->state();
            ?>
");

            // Get pointer
            constState = (const <?php 
            echo $gt->state();
            ?>
 *) constStatePtr.get_glaPtr();

            // Put back in container
            constStatePtr.swap(tempState);

<?php 
        }
        // if GT has a constant state
        ?>
            // Construct new GT
            <?php 
        echo $gtVar;
        ?>
 = new <?php 
        echo $gt;
        echo $cstStr;
        ?>
;

            // Play into container, so we can reuse it.
            GLAPtr newPtr( <?php 
        echo $gt->cHash();
        ?>
, <?php 
        echo $gtVar;
        ?>
 );
            QueryIDSet qry = <?php 
        echo queryName($query);
        ?>
;
            gtStates.Insert(qry, newPtr);
        } // if we had to create a state
    } // if query <?php 
        echo queryName($query);
        ?>
 is running

<?php 
    }
    // foreach query
    ?>

    // Prepare output bitstring iterator
    MMappedStorage queries_out_store;
    Column queries_out_col( queries_out_store );
    BStringIterator queries_out(queries_out_col, queriesCovered);

    // Prepare output for all passthrough attributes
<?php 
    cgConstructColumns($allPassthrough, '_out');
    ?>

    // Prepare output for synthesized attributes
<?php 
    cgConstructColumns($allSynth, '_out');
    ?>

    // Queries Covered by each passthrough attribute
<?php 
    foreach ($allPassthrough as $attr) {
        $type = $attr->type();
        $nullable = $type->is('nullable');
        $cstr = "";
        if ($nullable) {
            $cstr = "(GrokitNull::Value)";
        }
        ?>
    <?php 
        echo $type;
        ?>
 <?php 
        echo $attr->name();
        ?>
_out_default<?php 
        echo $cstr;
        ?>
;
    QueryIDSet <?php 
        echo $attr->name();
        ?>
_out_queries;
<?php 
        foreach ($queries as $query => $info) {
            if (in_array($attr, $info['pass'])) {
                ?>
    <?php 
                echo $attr->name();
                ?>
_out_queries.Union(<?php 
                echo queryName($query);
                ?>
);
<?php 
            }
            // if attribute is part of this query
        }
        // foreach query
    }
    // foreach passthrough attribute
    ?>

    // Queries covered by each synthesized attribute
<?php 
    foreach ($allSynth as $attr) {
        $type = $attr->type();
        $nullable = $type->is('nullable');
        $cstr = "";
        if ($nullable) {
            $cstr = "(GrokitNull::Value)";
        }
        ?>
    <?php 
        echo $type;
        ?>
 <?php 
        echo $attr->name();
        ?>
_out_default<?php 
        echo $cstr;
        ?>
;
    QueryIDSet <?php 
        echo $attr->name();
        ?>
_out_queries;
<?php 
        foreach ($queries as $query => $info) {
            if (in_array($attr, $info['output'])) {
                ?>
    <?php 
                echo $attr->name();
                ?>
_out_queries.Union(<?php 
                echo queryName($query);
                ?>
);
<?php 
            }
            // if attribute is part of this query
        }
        // foreach query
    }
    // foreach passthrough attribute
    ?>

    // Define constants used in expressions
<?php 
    foreach ($queries as $query => $info) {
        $input = $info['expressions'];
        cgDeclareConstants($input);
    }
    // foreach query
    ?>

    // Profiling information
    int64_t numTuplesIn = 0;
    int64_t numTuplesOut = 0;

#ifdef PER_QUERY_PROFILE
<?php 
    foreach ($queries as $query => $info) {
        ?>
    int64_t numTuplesIn_<?php 
        echo queryName($query);
        ?>
 = 0;
    int64_t numTuplesOut_<?php 
        echo queryName($query);
        ?>
 = 0;
<?php 
    }
    // foreach query
    ?>
#endif // PER_QUERY_PROFILE

    // Define should_iterate and is_done for each query
<?php 
    $iterVars = [];
    foreach ($queries as $query => $info) {
        $iterVar = 'should_process_' . queryName($query);
        $iterVars[] = $iterVar;
        ?>
    bool <?php 
        echo $iterVar;
        ?>
 = true;
<?php 
    }
    $iterExpr = implode(' || ', $iterVars);
    ?>

<?php 
    foreach ($queries as $query => $info) {
        // Call StartChunk() on iterable GTs
        $gt = $info['gt'];
        $gtVar = $gtVars[$query];
        if ($gt->iterable()) {
            ?>
    if (queriesToRun.Overlaps(<?php 
            echo queryName($query);
            ?>
)) {
        <?php 
            echo $gtVar;
            ?>
->StartChunk();
    }
<?php 
        }
    }
    ?>

    do {
<?php 
    cgAccessColumns($attMap, 'input', $wpName);
    ?>

    // Prepare input bitstring iterator
    BStringIterator queries_in;
    input.SwapBitmap(queries_in);
    
    numTuplesIn = 0;
#ifdef PER_QUERY_PROFILE
<?php 
    foreach ($queries as $query => $info) {
        ?>
    numTuplesIn_<?php 
        echo queryName($query);
        ?>
 = 0;
<?php 
    }
    // foreach query
    ?>
#endif // PER_QUERY_PROFILE

    // Begin processing tuples
    while( !queries_in.AtEndOfColumn() ) {
        numTuplesIn++;
        QueryIDSet qry = queries_in.GetCurrent();
        qry.Intersect(queriesToRun);
        queries_in.Advance();

<?php 
    cgAccessAttributes($attMap);
    foreach ($queries as $query => $info) {
        $gt = $info['gt'];
        $input = $info['expressions'];
        $output = $info['output'];
        $iterVar = 'should_process_' . queryName($query);
        $gtVar = $gtVars[$query];
        $outputVars = [];
        foreach ($output as $attr) {
            $outputVars[] = $attr->name() . '_out';
        }
        $inputVals = [];
        foreach ($input as $expr) {
            $inputVals[] = $expr->value();
        }
        // Prepare the output text
        ob_start();
        ?>
            // Insert synthesized attributes
<?php 
        foreach ($allSynth as $attr) {
            ?>
            if( queriesToRun.Overlaps(<?php 
            echo $attr->name();
            ?>
_out_queries) ) {
				<?php 
            echo $attr->name();
            ?>
_out_Column_Out.Insert(<?php 
            echo $attr->name();
            ?>
_out);
            }
			else {
				<?php 
            echo $attr->name();
            ?>
_out_Column_Out.Insert(<?php 
            echo $attr->name();
            ?>
_out_default);
			}
			<?php 
            echo $attr->name();
            ?>
_out_Column_Out.Advance();
<?php 
        }
        // foreach synthesized attribute
        ?>

            // Insert passthrough attributes
<?php 
        foreach ($allPassthrough as $attr) {
            ?>
            if( queriesToRun.Overlaps(<?php 
            echo $attr->name();
            ?>
_out_queries) ) {
                <?php 
            echo $attr->name();
            ?>
_out_Column_Out.Insert(<?php 
            echo $attr->name();
            ?>
);
            }
			else {
				<?php 
            echo $attr->name();
            ?>
_out_Column_Out.Insert(<?php 
            echo $attr->name();
            ?>
_out_default);
			}
			<?php 
            echo $attr->name();
            ?>
_out_Column_Out.Advance();
<?php 
        }
        // foreach synthesized attribute
        ?>

                queries_out.Insert(<?php 
        echo queryName($query);
        ?>
);
                queries_out.Advance();

                numTuplesOut++;
#ifdef PER_QUERY_PROFILE
                numTuplesOut_<?php 
        echo queryName($query);
        ?>
++;
#endif // PER_QUERY_PROFILE
<?php 
        $outputText = ob_get_clean();
        ?>
        // Do Query [<?php 
        echo queryName($query);
        ?>
]
        if( <?php 
        echo $iterVar;
        ?>
 && qry.Overlaps(<?php 
        echo queryName($query);
        ?>
) ) {
#ifdef PER_QUERY_PROFILE
            numTuplesIn_<?php 
        echo queryName($query);
        ?>
++;
#endif // PER_QUERY_PROFILE
<?php 
        cgDeclarePreprocessing($input, 3);
        ?>

<?php 
        if ($gt->result_type() == 'single') {
            ?>
            if( <?php 
            echo $gtVar;
            ?>
->ProcessTuple(<?php 
            echo implode(', ', $inputVals);
            ?>
, <?php 
            echo implode(', ', $outputVars);
            ?>
) ) {
<?php 
            echo $outputText;
            ?>

            } // if GT produced output for this tuple
<?php 
        } else {
            // if gt result type is single
            ?>
            <?php 
            echo $gtVar;
            ?>
->ProcessTuple(<?php 
            echo implode(', ', $inputVals);
            ?>
);

            while( <?php 
            echo $gtVar;
            ?>
->GetNextResult(<?php 
            echo implode(', ', $outputVars);
            ?>
) ) {
<?php 
            echo $outputText;
            ?>

            } // while GT has output for this tuple
<?php 
        }
        // if gt result type is multi
        ?>
        } // if <?php 
        echo queryName($query);
        ?>
 active in tuple

<?php 
    }
    // foreach query
    cgAdvanceAttributes($attMap, 2);
    ?>
    } // while we have tuples to process

<?php 
    foreach ($queries as $query => $info) {
        $gt = $info['gt'];
        $iterVar = 'should_process_' . queryName($query);
        $gtVar = $gtVars[$query];
        if ($gt->iterable()) {
            ?>
    if (<?php 
            echo $iterVar;
            ?>
) {
        <?php 
            echo $iterVar;
            ?>
 = <?php 
            echo $gtVar;
            ?>
->ShouldIterate();
    }

<?php 
        } else {
            ?>
    <?php 
            echo $iterVar;
            ?>
 = false;
<?php 
        }
    }
    //foreach query
    ?>

    // Put columns back into chunk
<?php 
    cgPutbackColumns($attMap, 'input', $wpName);
    ?>

    // Put bitmap back into chunk
    queries_in.Done();
    input.SwapBitmap(queries_in);

    } while(<?php 
    echo $iterExpr;
    ?>
);

    // Tell GTs that need to know about the chunk boundary
<?php 
    foreach ($queries as $query => $info) {
        $gt = $info['gt'];
        $gtVar = $gtVars[$query];
        if ($gt->chunk_boundary()) {
            ?>
    <?php 
            echo $gtVar;
            ?>
->ChunkBoundary();
<?php 
        }
        // if GT cares about chunk boundaries
    }
    // foreach query
    ?>



    // Finalize the output iterators and put the columns into the output chunk
    queries_out.Done();
    output.SwapBitmap(queries_out);

<?php 
    foreach ($allPassthrough as $attr) {
        ?>
    if( queriesToRun.Overlaps(<?php 
        echo $attr->name();
        ?>
_out_queries) ) {
        Column temp;
        <?php 
        echo $attr->name();
        ?>
_out_Column_Out.Done(temp);
        output.SwapColumn(temp, <?php 
        echo $attr->slot();
        ?>
);
    }
<?php 
    }
    // foreach passthrough attribute
    ?>

<?php 
    foreach ($allSynth as $attr) {
        ?>
    if( queriesToRun.Overlaps(<?php 
        echo $attr->name();
        ?>
_out_queries) ) {
        Column temp;
        <?php 
        echo $attr->name();
        ?>
_out_Column_Out.Done(temp);
        output.SwapColumn(temp, <?php 
        echo $attr->slot();
        ?>
);
    }
<?php 
    }
    // foreach passthrough attribute
    ?>

    PROFILING2_END;

    // Profiling
    PCounterList counterList;
    PCounter totalCntIn("tpi", numTuplesIn, "<?php 
    echo $wpName;
    ?>
");
    counterList.Append(totalCntIn);
    PCounter totalCntOut("tpo", numTuplesOut, "<?php 
    echo $wpName;
    ?>
");
    counterList.Append(totalCntOut);

#ifdef PER_QUERY_PROFILE
<?php 
    foreach ($queries as $query => $info) {
        ?>
    if( queriesToRun.Overlaps(<?php 
        echo queryName($query);
        ?>
) ) {
        PCounter cntIn("tpi <?php 
        echo queryName($query);
        ?>
", numTuplesIn_<?php 
        echo queryName($query);
        ?>
, "<?php 
        echo $wpName;
        ?>
");
        counterList.Append(cntIn);
        PCounter cntOut("tpo <?php 
        echo queryName($query);
        ?>
", numTuplesOut_<?php 
        echo queryName($query);
        ?>
, "<?php 
        echo $wpName;
        ?>
");
        counterList.Append(cntOut);
    }
<?php 
    }
    // foreach query
    ?>
#endif // PER_QUERY_PROFILE

    PROFILING2_SET(counterList, "<?php 
    echo $wpName;
    ?>
");

    // Return result and exit
    GTProcessChunkRez gtRez(gtStates, output);
    result.swap(gtRez);

    return WP_PROCESS_CHUNK;
}
//+{"kind":"WPF", "name":"Process Chunk", "action":"end"}
<?php 
}
Beispiel #4
0
function PrintGenerate($wpName, $queries, $attMap)
{
    ?>

// module specifsic headers to allow separate compilation
#include <iostream>
#include <string.h>
#include "Profiling.h"

//+{"kind":"WPF", "name":"Process Chunk", "action":"start"}
extern "C"
int PrintWorkFunc_<?php 
    echo $wpName;
    ?>
 (WorkDescription &workDescription, ExecEngineData &result) {

    // get the work description
    PrintWorkDescription myWork;
    myWork.swap (workDescription);
    Chunk &input = myWork.get_chunkToPrint ();
    QueryToFileMap& streams = myWork.get_streams();
    QueryToCounters& counters = myWork.get_counters();

    QueryIDSet queriesToRun = QueryExitsToQueries(myWork.get_whichQueryExits ());
    // prepare bitstring iterator
    Column inBitCol;
    BStringIterator queries;
    input.SwapBitmap (queries);

<?php 
    cgDeclareQueryIDs($queries);
    cgAccessColumns($attMap, 'input', $wpName);
    cgConstantInit($queries);
    ?>

    // for each query, define a stream variable
<?php 
    foreach ($queries as $query => $val) {
        $type = $val["type"];
        if ($type == 'json') {
            // Need some extra variables
            ?>
    Json::Value json;
    Json::Value jsonRow;
    Json::FastWriter jsonWriter;
    std::string jsonString;
<?php 
        }
        // if type is json
        ?>

    PrintFileObj& pfo_<?php 
        echo queryName($query);
        ?>
 = streams.Find(<?php 
        echo queryName($query);
        ?>
);
    DistributedCounter* counter_<?php 
        echo queryName($query);
        ?>
 = counters.Find(<?php 
        echo queryName($query);
        ?>
);
    FILE* file_<?php 
        echo queryName($query);
        ?>
 = pfo_<?php 
        echo queryName($query);
        ?>
.get_file();
    const char * DELIM_<?php 
        echo queryName($query);
        ?>
 = "<?php 
        echo $type == 'json' ? ',' : $val["separator"];
        ?>
";
#ifdef PER_QUERY_PROFILE
    size_t n_tuples_<?php 
        echo queryName($query);
        ?>
 = 0;
#endif // PER_QUERY_PROFILE
<?php 
    }
    // foreach query
    ?>
    // PRINTING
    constexpr const size_t BUFFER_LENGTH = 10 * 1024 * 1024; // 10 MB
    char buffer[BUFFER_LENGTH]; // ALIN, CHANGE THIS TO A DEFINED CONSTANT

    PROFILING2_START;
    size_t n_tuples = 0;
    while (!queries.AtEndOfColumn ()){
        ++n_tuples;
        QueryIDSet qry;
        qry = queries.GetCurrent();
        qry.Intersect(queriesToRun);
        queries.Advance();

<?php 
    cgAccessAttributes($attMap);
    foreach ($queries as $query => $val) {
        ?>
        // execute <?php 
        echo queryName($query);
        ?>
 code
        if (qry.Overlaps(<?php 
        echo queryName($query);
        ?>
) && counter_<?php 
        echo queryName($query);
        ?>
->Decrement(1)>=0){
<?php 
        cgPreprocess($val);
        ?>
#ifdef PER_QUERY_PROFILE
            ++n_tuples_<?php 
        echo queryName($query);
        ?>
;
#endif // PER_QUERY_PROFILE
            int curr=0; // the position where we write the next attribute

<?php 
        if ($type == 'json') {
            ?>
            jsonRow = Json::Value(Json::arrayValue);

<?php 
            foreach ($val["expressions"] as $exp) {
                ?>
            json = Json::Value(Json::nullValue);
            ToJson(<?php 
                echo $exp->value();
                ?>
, json);
            jsonRow.append(json);
<?php 
            }
            // for each expression
            ?>

            jsonString = jsonWriter.write(jsonRow);
            fprintf(file_<?php 
            echo queryName($query);
            ?>
, "%s,", jsonString.c_str());
<?php 
        } else {
            if ($type == 'csv') {
                foreach ($val["expressions"] as $exp) {
                    ?>
            curr += ToString(<?php 
                    echo $exp->value();
                    ?>
,buffer+curr);
            curr += sprintf(buffer + (curr-1), "%s", DELIM_<?php 
                    echo queryName($query);
                    ?>
) - 1;
<?php 
                }
                // for each expression
                ?>

            // Replace the last comma with a newline
            buffer[curr-1]='\n';

            // Null terminate the string
            buffer[curr]='\0';

            // Now we print the buffer
            fprintf(file_<?php 
                echo queryName($query);
                ?>
, "%s", buffer);
<?php 
            }
        }
        // if output file is csv
        ?>
        }
<?php 
    }
    // for each query
    cgAdvanceAttributes($attMap);
    ?>
    }

<?php 
    cgPutbackColumns($attMap, 'input', $wpName);
    ?>

    PROFILING2_END;

    PCounterList counterList;
    PCounter totalCnt("tpi", n_tuples, "<?php 
    echo $wpName;
    ?>
");
    counterList.Append(totalCnt);

#ifdef PER_QUERY_PROFILE
    // add query counters to list
<?php 
    foreach ($queries as $query => $val) {
        ?>
    {
        PCounter cnt("<?php 
        echo queryName($query);
        ?>
", n_tuples_<?php 
        echo queryName($query);
        ?>
, "<?php 
        echo $wpName;
        ?>
");
        counterList.Append(cnt);
    }
<?php 
    }
    ?>
#endif // PER_QUERY_PROFILE

    PROFILING2_SET(counterList, "<?php 
    echo $wpName;
    ?>
");

    // just return some arbitrary value... don't worry about reconstructing the chunk
    return WP_PROCESS_CHUNK;
}
//+{"kind":"WPF", "name":"Process Chunk", "action":"end"}

//+{"kind":"WPF", "name":"Finalize", "action":"start"}
extern "C"
int PrintFinalizeWorkFunc_<?php 
    echo $wpName;
    ?>
 (WorkDescription &workDescription, ExecEngineData &result) {

    PrintFinalizeWorkDescription myWork;
    myWork.swap( workDescription );
    QueryToFileMap& streams = myWork.get_streams();

    QueryIDSet queriesToRun = QueryExitsToQueries(myWork.get_whichQueryExits());

<?php 
    cgDeclareQueryIDs($queries);
    ?>

    // For each query, define a stream variable
<?php 
    $jsonVarsDefined = false;
    foreach ($queries as $query => $val) {
        $type = $val['type'];
        if ($type == 'json' && !$jsonVarsDefined) {
            $jsonVarsDefined = true;
            ?>
    Json::Value json;
    Json::FastWriter jsonWriter;
    std::string jsonString;
<?php 
        }
        // if we need to define extra json vars
        ?>
    PrintFileObj& pfo_<?php 
        echo queryName($query);
        ?>
 = streams.Find(<?php 
        echo queryName($query);
        ?>
);
    FILE* file_<?php 
        echo queryName($query);
        ?>
 = pfo_<?php 
        echo queryName($query);
        ?>
.get_file();
<?php 
    }
    // for each query
    ?>

<?php 
    foreach ($queries as $query => $val) {
        $type = $val['type'];
        if ($type == 'json') {
            ?>
    // Set up the types array
    json = Json::Value(Json::arrayValue);

<?php 
            foreach ($val['expressions'] as $exp) {
                $describer = $exp->type()->describer('json');
                grokit_assert(is_callable($describer), 'Invalid JSON describer for type ' . $exp->type());
                ?>
    {
        Json::Value tmp;
        <?php 
                $describer('tmp');
                ?>
        json.append(tmp);
    }
<?php 
            }
            // for each expression
            ?>

    jsonString = jsonWriter.write(json);
    // Last character is a newline, remove it
    jsonString.erase(jsonString.size() - 1, 1);

    // End the content section and write out the types
    fseek(file_<?php 
            echo queryName($query);
            ?>
, -1, SEEK_CUR); // overwrite the last comma
    fprintf(file_<?php 
            echo queryName($query);
            ?>
, " ], \"types\": %s }", jsonString.c_str());

<?php 
        }
        // if type is json
    }
    // for each query
    ?>

    return WP_FINALIZE;
}
//+{"kind":"WPF", "name":"Finalize", "action":"end"}
<?php 
}
Beispiel #5
0
function SelectionGenerate($wpName, $queries, $attMap)
{
    //echo PHP_EOL . '/*' . PHP_EOL;
    //print_r($wpName);
    //print_r($queries);
    //print_r($attMap);
    //echo PHP_EOL . '*/' . PHP_EOL;
    ?>

// module specific headers to allow separate compilation
#include "GLAData.h"
#include "Errors.h"

//+{"kind":"WPF", "name":"Pre-Processing", "action":"start"}
extern "C"
int SelectionPreProcessWorkFunc_<?php 
    echo $wpName;
    ?>

(WorkDescription& workDescription, ExecEngineData& result) {
    SelectionPreProcessWD myWork;
    myWork.swap(workDescription);
    QueryExitContainer& queries = myWork.get_whichQueryExits();
    QueryToGLASContMap & requiredStates = myWork.get_requiredStates();

    QueryToGLAStateMap constStates;
<?php 
    cgDeclareQueryIDs($queries);
    ?>

<?php 
    foreach ($queries as $query => $info) {
        $gf = $info['gf'];
        if (!is_null($gf) && $gf->has_state()) {
            $state = $gf->state();
            if ($state->configurable()) {
                $carg = $info['cargs'];
                echo '    // JSON Configuration for query ' . queryName($query) . PHP_EOL;
                $carg->init();
                echo PHP_EOL;
            }
            // if gf const state is configurable
        }
        // if gf has state
    }
    //foreach query
    ?>

    FOREACH_TWL(iter, queries) {
<?php 
    foreach ($queries as $query => $val) {
        ?>
        if( iter.query == <?php 
        echo queryName($query);
        ?>
) {
<?php 
        if ($val['gf'] !== null) {
            // This is a generalized filter
            $gf = $val['gf'];
            $given_states = $val['states'];
            if ($gf->has_state()) {
                $cstArgs = [];
                $state = $gf->state();
                // If the state is configurable, give it the JSON carg
                if ($state->configurable()) {
                    $carg = $query['cargs'];
                    $cstArgs[] = $carg->name();
                }
                // if gf state is configurable
                if (\count($given_states) > 0) {
                    ?>
            FATALIF(!requiredStates.IsThere(<?php 
                    echo queryName($query);
                    ?>
),
                "No required states received for query that declared required states");
            GLAStateContainer& givenStates = requiredStates.Find(<?php 
                    echo queryName($query);
                    ?>
);
            givenStates.MoveToStart();
            GLAPtr reqTemp;
<?php 
                    foreach ($givenStates as $gs) {
                        $cstArgs[] = $gs->name();
                        ?>
            // Extract state from waypoint[<?php 
                        echo $gs->waypoint();
                        ?>
]
            <?php 
                        echo $gs->type();
                        ?>
 * <?php 
                        echo $gs->name();
                        ?>
 = nullptr;
            reqTemp.Swap(givenStates.Current());
            FATALIF( reqTemp.get_glaType() != <?php 
                        echo $gs->type()->cHash();
                        ?>
,
                "Got different type than expected for required state of type <?php 
                        echo $gs > type();
                        ?>
");
            <?php 
                        echo $gs->name();
                        ?>
 = (<?php 
                        echo $gs->type();
                        ?>
 *) reqTemp.get_glaPtr();
            reqTemp.swap(givenStates.Current());
            givenStates.Advance();
<?php 
                    }
                    // foreach given state
                }
                // if we have given states
                $cstStr = \count($cstArgs) > 0 ? '(' . implode(', ', $cstArgs) . ')' : '';
                ?>
            <?php 
                echo $state;
                ?>
 * temp = new <?php 
                echo $state;
                echo $cstStr;
                ?>
;
            GLAPtr newPtr( <?php 
                echo $state->cHash();
                ?>
, (void *) temp );
            QueryID qryID = <?php 
                echo queryName($query);
                ?>
;
            constStates.Insert(qryID, newPtr);
<?php 
            }
            // if gf has state
        }
        // if( $val['gf'] !== null )
        ?>
        } // if <?php 
        echo queryName($query);
        ?>
 is current query
<?php 
    }
    // foreach query
    ?>
  } END_FOREACH;

  SelectionPreProcessRez myRez( constStates );
  myRez.swap(result);

  return WP_PREPROCESSING; // for PreProcess
}
//+{"kind":"WPF", "name":"Pre-Processing", "action":"end"}

//+{"kind":"WPF", "name":"Process Chunk", "action":"start"}
extern "C"
int SelectionProcessChunkWorkFunc_<?php 
    echo $wpName;
    ?>
 (WorkDescription &workDescription, ExecEngineData &result) {
    // go to the work description and get the input chunk
    SelectionProcessChunkWD myWork;
    myWork.swap (workDescription);
    Chunk &input = myWork.get_chunkToProcess ();
    QueryToGLAStateMap& constStates = myWork.get_constStates();

    PROFILING2_START;

    QueryIDSet queriesToRun = QueryExitsToQueries(myWork.get_whichQueryExits ());
<?php 
    cgDeclareQueryIDs($queries);
    cgAccessColumns($attMap, 'input', $wpName);
    // Declare the constants needed by the filters and synth expressions.
    foreach ($queries as $query => $val) {
        ?>
    // Constants for query <?php 
        echo queryName($query);
        ?>
:
<?php 
        $filters = $val['filters'];
        $synths = $val['synths'];
        cgDeclareConstants($filters);
        cgDeclareConstants($synths);
    }
    // foreach query
    ?>

    // prepare bitstring iterator
    Column inBitCol;
    BStringIterator queries;
    input.SwapBitmap (queries);

    // creating storage for syhthesized attributes
<?php 
    foreach ($queries as $query => $val) {
        $synList = $val['synths'];
        foreach ($synList as $att => $syn) {
            ?>
    MMappedStorage <?php 
            echo attStorage($att);
            ?>
;
    Column <?php 
            echo attCol($att);
            ?>
(<?php 
            echo attStorage($att);
            ?>
);
    <?php 
            echo attIteratorType($att);
            ?>
 colI_<?php 
            echo $att;
            ?>
(<?php 
            echo attCol($att);
            ?>
);
    <?php 
            echo attType($att);
            ?>
 <?php 
            echo $att;
            ?>
;
<?php 
        }
        // foreach synthesized attribute
    }
    // foreach query
    ?>


<?php 
    foreach ($queries as $query => $val) {
        $givenStates = $val['states'];
        $gf = $val['gf'];
        $cargs = $val['cargs'];
        grokit_assert($gf !== null || count($givenStates) == 0, 'We were given states for query ' . $query . ' when we have no GF!');
        if (!is_null($gf) && $gf->has_state()) {
            $state = $gf->state();
            $stateName = 'cst_state_' . queryName($query);
            $constMod = $state->mutable() ? '' : 'const ';
            ?>
    // Extracting constant state for query <?php 
            echo queryName($query);
            ?>
    FATALIF(!constStates.IsThere(<?php 
            echo queryName($query);
            ?>
), "No constant state found for query <?php 
            echo queryName($query);
            ?>
.");
    <?php 
            echo $constMod;
            echo $state;
            ?>
 * <?php 
            echo $stateName;
            ?>
 = nullptr;
    {
        GLAState& curState = constStates.Find(<?php 
            echo queryName($query);
            ?>
);
        GLAPtr tmp;
        tmp.swap(curState);
        FATALIF( tmp.get_glaType() != <?php 
            echo $state->cHash();
            ?>
,
            "Got different type than expected for constant state of type <?php 
            echo $state;
            ?>
");
        <?php 
            echo $stateName;
            ?>
 = (<?php 
            echo $constMod;
            echo $state;
            ?>
 *) tmp.get_glaPtr();
        tmp.swap(curState);
    }

<?php 
        }
        // if gf requires constant state
        if ($gf !== null) {
            $ctrArgs = [];
            if ($gf->configurable()) {
                echo '    // JSON initialiser for query ' . queryName($query) . PHP_EOL;
                $cargs->init();
                echo PHP_EOL;
                $ctrArgs[] = $cargs->name();
            }
            if ($gf->has_state()) {
                $ctrArgs[] = '*' . $stateName;
            }
            $ctrStr = \count($ctrArgs) > 0 ? '(' . implode(', ', $ctrArgs) . ')' : '';
            ?>
    // Construct GF for query <?php 
            echo queryName($query);
            ?>

    <?php 
            echo $gf->value();
            ?>
 <?php 
            echo queryName($query);
            ?>
_state<?php 
            echo $ctrStr;
            ?>
;
<?php 
        }
        // if we have a GF
    }
    // foreach query
    ?>

    MMappedStorage bitStore;
    Column outBitCol(bitStore);
    BStringIterator outQueries (outBitCol, queriesToRun);

#ifdef PER_QUERY_PROFILE
<?php 
    foreach ($queries as $query => $val) {
        ?>
    int64_t n_tuples_<?php 
        echo queryName($query);
        ?>
 = 0;
<?php 
    }
    // foreach query
    ?>
#endif // PER_QUERY_PROFILE

    int64_t numTuples = 0;
    while (!queries.AtEndOfColumn ()) {
        ++numTuples;
        QueryIDSet qry;
        qry = queries.GetCurrent();
        qry.Intersect(queriesToRun);
        queries.Advance();

        //selection code for all the predicates
<?php 
    cgAccessAttributes($attMap);
    foreach ($queries as $query => $val) {
        $gf = $val['gf'];
        $filters = $val['filters'];
        $synths = $val['synths'];
        $stateName = queryName($query) . '_state';
        $filterVals = array_map(function ($expr) {
            return '(' . $expr . ')';
        }, $filters);
        if ($gf === null) {
            // Simple set of expressions.
            if (\count($filterVals) > 0) {
                $selExpr = implode(' && ', $filterVals);
            } else {
                $selExpr = 'true';
            }
        } else {
            // We have a GF
            $selExpr = "{$stateName}.Filter(" . implode(', ', $filterVals) . ")";
        }
        ?>
        // do <?php 
        echo queryName($query);
        ?>
:
<?php 
        foreach ($synths as $att => $syn) {
            ?>
        <?php 
            echo attType($att);
            ?>
 <?php 
            echo $att;
            ?>
;
<?php 
        }
        // foreach synthesized attribute
        ?>
        if( qry.Overlaps(<?php 
        echo queryName($query);
        ?>
) ) {
#ifdef PER_QUERY_PROFILE
            ++numTuples_<?php 
        echo queryName($query);
        ?>
;
#endif // PER_QUERY_PROFILE
<?php 
        cgDeclarePreprocessing($filters, 2);
        ?>
            if( <?php 
        echo $selExpr;
        ?>
 ) {
                // compute synthesized
<?php 
        cgDeclarePreprocessing($synths, 3);
        foreach ($synths as $att => $expr) {
            ?>
            <?php 
            echo $att;
            ?>
 = <?php 
            echo $expr;
            ?>
;
<?php 
        }
        //foreach synthesized attribute
        ?>
            } else {
                qry.Difference(<?php 
        echo queryName($query);
        ?>
);
            }
        }
<?php 
        foreach ($synths as $att => $syn) {
            ?>
            colI_<?php 
            echo $att;
            ?>
.Insert(<?php 
            echo $att;
            ?>
);
            colI_<?php 
            echo $att;
            ?>
.Advance();
<?php 
        }
        // foreach synthesized attribute
    }
    // foreach query
    ?>
        outQueries.Insert(qry);
        outQueries.Advance();

<?php 
    cgAdvanceAttributes($attMap);
    ?>
    } // while we still have tuples remaining

    // finally, if there were any results, put the data back in the chunk
<?php 
    cgPutbackColumns($attMap, 'input', $wpName);
    foreach ($queries as $query => $val) {
        $synths = $val['synths'];
        ?>
    if (<?php 
        echo queryName($query);
        ?>
.Overlaps(queriesToRun)) {
<?php 
        foreach ($synths as $att => $expr) {
            ?>
        colI_<?php 
            echo $att;
            ?>
.Done(<?php 
            echo attCol($att);
            ?>
);
        input.SwapColumn(<?php 
            echo attCol($att);
            ?>
, <?php 
            echo attSlot($att);
            ?>
);
<?php 
        }
        //foreach synthesized attribute
        ?>
    } // If <?php 
        echo queryName($query);
        ?>
 overlaps queriesToRun
<?php 
    }
    // foreach query
    ?>

    // put in the output bitmap
    outQueries.Done ();
    input.SwapBitmap (outQueries);

    // Finish performance counters
    PROFILING2_END;

    PCounterList counterList;
    PCounter totalCnt("tpi", numTuples, "<?php 
    echo $wpName;
    ?>
");
    counterList.Append(totalCnt);
    PCounter tplOutCnt("tpo", numTuples, "<?php 
    echo $wpName;
    ?>
");
    counterList.Append(tplOutCnt);

#ifdef PER_QUERY_PROFILE
<?php 
    foreach ($queries as $query => $val) {
        $filters = $val['filters'];
        ?>
    if( <?php 
        echo queryName($query);
        ?>
.Overlaps(queriesToRun)) {
        PCounter cnt("<?php 
        echo queryName($query);
        ?>
", numTuples_<?php 
        echo queryName($query);
        ?>
, "<?php 
        echo $wpName;
        ?>
");
        counterList.Append(cnt);
    }
<?php 
    }
    // foreach query
    ?>
#endif // PER_QUERY_PROFILE

    PROFILING2_SET(counterList, "<?php 
    echo $wpName;
    ?>
");

    ChunkContainer tempResult (input);
    tempResult.swap (result);

    return WP_PROCESS_CHUNK; // For Process Chunk
}
//+{"kind":"WPF", "name":"Process Chunk", "action":"end"}

<?php 
}
Beispiel #6
0
function JoinRHS($wpName, $jDesc)
{
    ?>

//+{"kind":"WPF", "name":"RHS Hash", "action":"start"}
extern "C"
int JoinRHSWorkFunc_<?php 
    echo $wpName;
    ?>
 (WorkDescription &workDescription, ExecEngineData &result) {

    double start_time = global_clock.GetTime();
    PROFILING2_START;

    // this is the area where all of the intermediate, serialized records are stored
    SerializedSegmentArray serializedSegments [NUM_SEGS];

    // this is the area where all of the records are serialized to;
    // 10K bytes are initially used for this
    char *serializeHere = (char *) malloc (10000);
    int storageSize = 10000;

    // go to the work description and get the input chunk
    JoinRHSWorkDescription myWork;
    myWork.swap (workDescription);
    Chunk &input = myWork.get_chunkToProcess ();

    // get the waypoint identifier
    unsigned int wayPointID = myWork.get_wayPointID ();

    QueryIDSet queriesToRun = QueryExitsToQueries(myWork.get_whichQueryExits ());

<?php 
    cgAccessColumns($jDesc->attribute_queries_RHS, 'input', $wpName);
    ?>

    // prepare bitstring iterator
    Column inBitCol;
    BStringIterator queries;
    input.SwapBitmap (queries);

    int totalNum = 0; // counter for the tuples processed

    // now actually hash all of the tuples!
    while (!queries.AtEndOfColumn ()){
        QueryIDSet qry;
        qry = queries.GetCurrent();
        qry.Intersect(queriesToRun);
        queries.Advance();

        // extract values of attributes from streams
<?php 
    cgAccessAttributes($jDesc->attribute_queries_RHS);
    ?>

         if (qry.IsEmpty()){
<?php 
    cgAdvanceAttributes($jDesc->attribute_queries_RHS);
    ?>
             continue;
         }

        totalNum++;

<?php 
    foreach ($jDesc->query_classes_hash as $qClass) {
        $attOrder = [];
        foreach ($qClass->att_queries as $att => $qrys) {
            $attr = lookupAttribute($att);
            $attOrder[$att] = $attr->slot();
        }
        asort($attOrder);
        ?>
        // Dealing with join attributes <?php 
        echo implode(",", $qClass->att_list);
        ?>

        if (qry.Overlaps(QueryIDSet(<?php 
        echo $qClass->qClass;
        ?>
, true))) {

            HT_INDEX_TYPE hashValue = HASH_INIT;
    <?php 
        foreach ($qClass->rhs_keys as $att) {
            ?>
            hashValue = CongruentHash(Hash(<?php 
            echo $att;
            ?>
), hashValue);
    <?php 
        }
        /*foreach attribute*/
        ?>

            // figure out which of the hash buckets it goes into
            unsigned int index = WHICH_SEGMENT (hashValue);

            // and serialize the record!  Begin with the bitstring.
            // TBD TBD SS: check if Bitstring takes value that way !
            Bitstring myInBString(<?php 
        echo $qClass->qClass;
        ?>
, true);
            myInBString.Intersect(qry);

            int bytesUsed = sizeof(Bitstring);

            // Make sure we have the storage...
            if (bytesUsed > storageSize) {
                storageSize = bytesUsed;
                free (serializeHere);
                serializeHere = (char *) malloc (storageSize);
            }

            // do the serialization...
            void *location = (void*)&myInBString;

            // remember the serialized value
            serializedSegments[index].StartNew (WHICH_SLOT (hashValue), wayPointID, 1, location, bytesUsed);

            // now, go thru all of the attributes that are used
<?php 
        foreach ($attOrder as $att => $slot) {
            $qrys = $qClass->att_queries->{$att};
            ?>
            if (myInBString.Overlaps(QueryIDSet(<?php 
            echo $qrys;
            ?>
, true))){

                //bytesUsed = <?php 
            echo attSerializedSize($att, $att);
            ?>
;
		bytesUsed = SerializedSize(<?php 
            echo $att;
            ?>
);
                if (bytesUsed > storageSize) {
                    storageSize = bytesUsed;
                    free (serializeHere);
                    serializeHere = (char *) malloc (storageSize);
                }

                // and record the serialized value
                //location =  <?php 
            echo attOptimizedSerialize($att, $att, "serializeHere");
            ?>
;
		Serialize(serializeHere, <?php 
            echo $att;
            ?>
);
                serializedSegments[index].Append (<?php 
            echo $slot;
            ?>
,(void *) serializeHere, bytesUsed);
            }
    <?php 
        }
        /*foreach attribute*/
        ?>
        }

<?php 
    }
    /*foreach query class*/
    /* Is this correct. Should it be inside the loop for the class? */
    cgAdvanceAttributes($jDesc->attribute_queries_RHS);
    ?>
    }

    // now we are done serializing the chunk
    free (serializeHere);

    // so actually do the hashing... first set up the list of the guys we want to hash
    int theseAreOK [NUM_SEGS];
    for (int i = 0; i < NUM_SEGS; i++) {
        theseAreOK[i] = 1;
    }

    // this is the set of sample collisions taken from the over-full segments
    HashSegmentSample mySamples;

    // now go through and, one-at-a-time, add the data to each table segment
    for (int i = 0; i < NUM_SEGS; i++) {
        // first get a segment to add data to
        HashTableSegment checkedOutCopy;
        int whichOne = myWork.get_centralHashTable ().CheckOutOne (theseAreOK, checkedOutCopy);
        theseAreOK[whichOne] = 0;

        // now add the data
        HashSegmentSample mySample;
        if (checkedOutCopy.Insert (serializedSegments[whichOne], mySample)) {

            // if we are in here, it means that the segment was over-full, so note that we will
            // need to empty it out... we record all of the samples
            mySamples.MoveToFinish ();
            mySample.MoveToStart ();
            mySamples.SwapRights (mySample);
        }

        // and then put the segment back in the hash table
        myWork.get_centralHashTable ().CheckIn (whichOne);
    }

<?php 
    cgPutbackColumns($jDesc->attribute_queries_RHS, 'input', $wpName);
    ?>

    PROFILING2_END;

    PROFILING(start_time, "<?php 
    echo $wpName;
    ?>
", "RHS_hash", "%d", totalNum);
    PROFILING(0.0, "HashTable", "fillrate", "%2.4f", HashTableSegment::globalFillRate*100.0);

    // Finish performance counters
    // Use the Set functionality in case we add additional counters later.
    PCounterList counterList;
    PCounter totalCnt("RHS", totalNum, "<?php 
    echo $wpName;
    ?>
");
    counterList.Append(totalCnt);
    PCounter globalCnt("jRHS", totalNum, "global");
    counterList.Append(globalCnt);

    PROFILING2_SET(counterList, "<?php 
    echo $wpName;
    ?>
");

    int64_t hFillRate = int64_t(HashTableSegment::globalFillRate * 1000);
    PROFILING2_INSTANT("hfr", hFillRate, "global");

    // now we are finally done!
    JoinHashResult myResult (mySamples);
    myResult.swap (result);
    return 0;

} // JoinRHSWorkFunc_<?php 
    echo $wpName;
    ?>
 function
//+{"kind":"WPF", "name":"RHS Hash", "action":"end"}

<?php 
}