Esempio n. 1
0
function cgExtractColumnFragment($att, $chunk, $start, $end, $wpName)
{
    ?>
            // extracting <?php 
    echo $att;
    ?>
:
            Column <?php 
    echo attCol($att);
    ?>
;

            if (<?php 
    echo attQrys($att);
    ?>
.Overlaps(queriesToRun)){
                <?php 
    echo $chunk;
    ?>
.SwapColumn(<?php 
    echo attCol($att);
    ?>
, <?php 
    echo attSlot($att);
    ?>
);
                if (! <?php 
    echo attCol($att);
    ?>
.IsValid()){
                    FATAL("Error: Column <?php 
    echo $att;
    ?>
 not found in <?php 
    echo $wpName;
    ?>
\n");
                }
            }
            <?php 
    echo attIteratorType($att);
    ?>
 <?php 
    echo attData($att);
    ?>
(<?php 
    echo attCol($att);
    ?>
/*, 8192*/, <?php 
    echo $start;
    ?>
, <?php 
    echo $end;
    ?>
);
<?php 
}
Esempio n. 2
0
function JoinLHS($wpName, $jDesc)
{
    $rhsAttOrder = [];
    foreach ($jDesc->hash_RHS_attr as $attr) {
        $att = lookupAttribute($attr);
        $rhsAttOrder[$att->slot()] = $attr;
    }
    ksort($rhsAttOrder);
    $jDesc->hash_RHS_attr = $rhsAttOrder;
    ?>

//+{"kind":"WPF", "name":"LHS Lookup", "action":"start"}
extern "C"
int JoinLHSWorkFunc_<?php 
    echo $wpName;
    ?>
(WorkDescription &workDescription, ExecEngineData &result) {

    double start_time = global_clock.GetTime();
    PROFILING2_START;
    // this is the area where all of the intermediate, serialized records are stored
    SerializedSegmentArray serializedSegments [NUM_SEGS];

    // this is the area where all of the records are serialized to;
    // 10K bytes are initially used for this
    char *serializeHere = (char *) malloc (10000);

    // this is the output chunk
    Chunk output;

    // go to the work description and get the input chunk
    JoinLHSWorkDescription myWork;
    myWork.swap (workDescription);
    Chunk &input = myWork.get_chunkToProcess ();

    // get the waypoint ID from the chunk
    int wayPointID = myWork.get_wayPointID ();

    QueryIDSet queriesToRun = QueryExitsToQueries(myWork.get_whichQueryExits ());

<?php 
    cgAccessColumns($jDesc->attribute_queries_LHS, 'input', $wpName);
    ?>

    BStringIterator myInBStringIter;
    input.SwapBitmap (myInBStringIter);

    // start the iterators for the output columns for LHS; used only if stillShallow = 0
<?php 
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $queries) {
        ?>
    <?php 
        echo attIteratorType($att);
        ?>
 <?php 
        echo $att;
        ?>
_Column_Out;
<?php 
    }
    /*foreach*/
    ?>

    // these manage the output columns that come from the RHS (now stored in the hash table)
<?php 
    cgConstructColumns(array_keys($jDesc->attribute_queries_RHS_copy));
    ?>

    // this is the ouput bitstring
    MMappedStorage myStore;
    Column bitmapOut (myStore);
    BStringIterator myOutBStringIter (bitmapOut, queriesToRun);

    // now we extract all of the hash table segments... after this, myEntries will hold them all
    HashTableView myView;
    myWork.get_centralHashTable ().EnterReader (myView);
    HashTableSegment myEntries[NUM_SEGS];
    myView.ExtractAllSegments (myEntries);

    // this tells us that we are "still shallow"---not making a deep copy of the LHS atts to the output
    int stillShallow = 1;

    // the bitstring that will be exracted from the hash table
    QueryIDSet *bitstringRHS = 0;

    QueryIDSet existsTarget(<?php 
    echo $jDesc->exists_target;
    ?>
, true);
    QueryIDSet notExistsTarget(<?php 
    echo $jDesc->not_exists_target;
    ?>
, true);

    // these are all of the attribute values that come from the hash table...
    // for each att we need a pointer as well as a dummy value that the pointer will be set to by default

<?php 
    foreach ($jDesc->attribute_queries_RHS as $att => $queries) {
        ?>
    QueryIDSet <?php 
        echo attQrys($att);
        ?>
_RHS(<?php 
        echo $queries;
        ?>
, true);
    <?php 
        echo attType($att);
        ?>
 <?php 
        echo $att;
        ?>
RHSShadow;
    <?php 
        echo attType($att);
        ?>
 *<?php 
        echo $att;
        ?>
RHS = NULL;
    <?php 
        echo attType($att);
        ?>
 <?php 
        echo $att;
        ?>
RHSobj;
<?php 
    }
    /*foreach*/
    ?>

    // now actually try to match up all of the tuples!
    int totalNum = 0;
    while (!myInBStringIter.AtEndOfColumn ()) { // TBD, probably this is not working TBD

        // counts how many matches for this query
        int numHits = 0;

        // extract values of attributes from streams
        // now go through the LHS input atts one at a time and extract if it is needed by an active query

        // see which queries match up
        QueryIDSet curBits = myInBStringIter.GetCurrent ();
        curBits.Intersect (queriesToRun);

        QueryIDSet exists; // keeps track of the queries for which a match is found
        QueryIDSet oldBitstringLHS; // last value of bistringLHS


        // if the input query is not empty
        if (!curBits.IsEmpty ()) {

            totalNum++;

            // compute the hash for LHS
            HT_INDEX_TYPE hashValue = HASH_INIT;
<?php 
    foreach ($jDesc->LHS_keys as $att) {
        ?>
            hashValue = CongruentHash(Hash(<?php 
        echo $att;
        ?>
_Column.GetCurrent()), hashValue);
<?php 
    }
    /*foreach*/
    ?>

            // figure out which of the hash buckets it goes into
            unsigned int index = WHICH_SEGMENT (hashValue);

            // now, go to that index and extract matching tuples!
            HT_INDEX_TYPE curSlot = WHICH_SLOT (hashValue);
            hashValue = curSlot;

            // this loops through all of the possible RHS hits
            while (1) {

                // this is the bitstring that will go in the output
                QueryIDSet bitstringLHS;

                // for safety (in case we look at a bitstring that spans multiple
                // entries that is not done being written by a concurrent writer)
                // empty out the inital bitstring
                ((QueryIDSet *) serializeHere)->Empty ();

                // give safe "shadow" values to all of the RHS attributes
<?php 
    foreach ($jDesc->hash_RHS_attr as $att) {
        ?>
                <?php 
        echo $att;
        ?>
RHS = &<?php 
        echo $att;
        ?>
RHSShadow;
<?php 
    }
    /*foreach*/
    ?>

                // here we go through and extract the atts one at a time from the hash
                // table.  Note that the atts must be extracted IN ORDER.  That is, the
                // bitstring is first, followed by the att mapped to the lowerest column
                // position, followed by the att mapped to the next lowest, and so on.

                // The Extract function pulls an attribute out of the hash table...
                int lenSoFar = 0, dummy, done;
                int lastLen = myEntries[index].Extract (serializeHere, curSlot, hashValue, wayPointID, BITMAP, dummy, done);

                // if we cannot find a bitstring, there was no tuple here, and we are done
                if (lastLen == 0) {
                    break;
                }

                // remember the bitstring
                bitstringRHS = (QueryIDSet *) serializeHere;
                lenSoFar += lastLen;

                // next look for other hashed attributes
<?php 
    foreach ($jDesc->hash_RHS_attr as $att) {
        ?>
                lastLen = myEntries[index].Extract (serializeHere + lenSoFar, curSlot, hashValue, wayPointID, <?php 
        echo attSlot($att);
        ?>
, dummy, done);

                // see if we got attribute
                if (lastLen > 0) {
                    Deserialize(serializeHere + lenSoFar, <?php 
        echo $att;
        ?>
RHSobj);
                    //<?php 
        echo attOptimizedDeserialize($att, $att . "RHSobj", "serializeHere", "lenSoFar");
        ?>
;
                    <?php 
        echo $att;
        ?>
RHS = &<?php 
        echo $att;
        ?>
RHSobj;
                    lenSoFar += lastLen;
                } else {
                    FATALIF(<?php 
        echo attQrys($att);
        ?>
_RHS.Overlaps(*bitstringRHS),
                        "Did not find attribute <?php 
        echo $att;
        ?>
 in active RHS tuple");
                }
<?php 
    }
    /*foreach*/
    ?>

                // see if we have any query matches
                bitstringRHS->Intersect (curBits);
                QueryIDSet qBits;
                //printf("TPLLLLL: cust_acctbal = %f    orders_custkey = %d   cust_custkey = %d\n", *customer_c_acctbalRHS, orders_o_custkey_Column.GetCurrent(), *customer_c_custkeyRHS);

<?php 
    foreach ($jDesc->queries_attribute_comparison as $qClass) {
        ?>
                // See if any query in query class is eligible for this comparision
                qBits = QueryIDSet(<?php 
        echo $qClass->qClass;
        ?>
, true);
                qBits.Intersect(*bitstringRHS);
                if (
                    !qBits.IsEmpty () &&

    <?php 
        foreach ($qClass->att_pairs as $pair) {
            ?>
                     *<?php 
            echo $pair->rhs;
            ?>
RHS == <?php 
            echo $pair->lhs;
            ?>
_Column.GetCurrent() &&
    <?php 
        }
        /*foreach pair*/
        ?>
                     1 )
                {
                    bitstringLHS.Union (qBits);
                }
<?php 
    }
    /*foreach query class*/
    ?>

                // if any of them hit...
                if (!bitstringLHS.IsEmpty ()) {

                    exists.Union(bitstringLHS);

                    numHits++;

                    // see if we need to move from shallow to deep
                    if (numHits == 2 && stillShallow) {

<?php 
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $qrys) {
        ?>
                        <?php 
        echo attData($att);
        ?>
_Out.CreateDeepCopy (<?php 
        echo attData($att);
        ?>
);
                        <?php 
        echo attData($att);
        ?>
_Out.Insert (<?php 
        echo attData($att);
        ?>
.GetCurrent());
                        <?php 
        echo attData($att);
        ?>
_Out.Advance();

<?php 
    }
    /*foreach*/
    ?>
                        stillShallow = 0;
                    }

                    // now, add all of the outputs over... first deal with the LHS input atts
                    // that get copied into output atts
                    if (!stillShallow) {
<?php 
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $qrys) {
        ?>
                        <?php 
        echo attData($att);
        ?>
_Out.Insert (<?php 
        echo attData($att);
        ?>
.GetCurrent());
                        <?php 
        echo attData($att);
        ?>
_Out.Advance();

<?php 
    }
    /*foreach*/
    ?>
                    }

                    // now, deal with the output atts that come from the hash table
<?php 
    foreach ($jDesc->attribute_queries_RHS_copy as $att => $qrys) {
        ?>
                    <?php 
        echo attData($att);
        ?>
_Out.Insert (*<?php 
        echo $att;
        ?>
RHS);
                    <?php 
        echo attData($att);
        ?>
_Out.Advance();
<?php 
    }
    /*foreach*/
    ?>


                    // finally, set the bitmap. We are one element behind
                    if (!oldBitstringLHS.IsEmpty()){
                         myOutBStringIter.Insert (oldBitstringLHS);
                                 myOutBStringIter.Advance ();
                    }
                    oldBitstringLHS=bitstringLHS;
                }  // empty bistring
            }
        }

    // compute the true exist queries
    QueryIDSet tmp = existsTarget;
    tmp.Intersect(exists);
    tmp.Intersect(curBits); // not needed but I'm paranoid

    // compute the true not exits queries
    QueryIDSet tmp2 = curBits;
    tmp2.Intersect(notExistsTarget);
    tmp2.Difference(exists);

    // now put everything in bitstringLHS
    oldBitstringLHS.Union(tmp);
    oldBitstringLHS.Union(tmp2);

    if (!oldBitstringLHS.IsEmpty()){
             myOutBStringIter.Insert (oldBitstringLHS);
                 myOutBStringIter.Advance ();
        }


        // at this point, we are done trying to join this tuple... any join results have been
        // written to the output columns.  Note that we don't have to advance in the output data
        // columns; if we are shallow, we don't touch the output columns.  If we are not shallow,
        // if there were no results, we have nothing to write.  HOWEVER, if we are shallow and
        // we did not get a match, we need to add an empty btstring
        if (stillShallow && numHits == 0) {
<?php 
    foreach ($jDesc->attribute_queries_RHS_copy as $att => $qrys) {
        ?>
            <?php 
        echo attType($att);
        ?>
 tmp_<?php 
        echo attData($att);
        ?>
;
            <?php 
        echo attData($att);
        ?>
_Out.Insert (tmp_<?php 
        echo attData($att);
        ?>
);
            <?php 
        echo attData($att);
        ?>
_Out.Advance();
<?php 
    }
    /*foreach*/
    ?>

        if (oldBitstringLHS.IsEmpty()){ // no not exist and no join match
           myOutBStringIter.Insert (oldBitstringLHS);
               myOutBStringIter.Advance ();
        }
        }

        // lastly, we need to advance in the INPUT tuples
<?php 
    foreach ($jDesc->attribute_queries_LHS as $att => $qrys) {
        ?>
        <?php 
        echo attData($att);
        ?>
.Advance();
<?php 
    }
    /*foreach*/
    ?>

        // advance the input bitstring
        myInBStringIter.Advance ();
    }

    // DONE!  So construct the output tuple

    // if we are still shallow, put the original data into the output
    if (stillShallow) {
<?php 
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $qrys) {
        ?>
        Column col_<?php 
        echo $att;
        ?>
;
        <?php 
        echo attData($att);
        ?>
.Done(col_<?php 
        echo $att;
        ?>
);
        output.SwapColumn (col_<?php 
        echo $att;
        ?>
, <?php 
        echo attSlot($att);
        ?>
);
<?php 
    }
    /*foreach*/
    ?>
    } else {
<?php 
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $qrys) {
        ?>
        Column col_<?php 
        echo $att;
        ?>
;
        <?php 
        echo attData($att);
        ?>
_Out.Done(col_<?php 
        echo $att;
        ?>
);
        output.SwapColumn (col_<?php 
        echo $att;
        ?>
, <?php 
        echo attSlot($att);
        ?>
);
<?php 
    }
    /*foreach*/
    ?>
    }

    {
<?php 
    foreach ($jDesc->attribute_queries_RHS_copy as $att => $qrys) {
        ?>
        Column col_<?php 
        echo $att;
        ?>
;
        <?php 
        echo attData($att);
        ?>
_Out.Done(col_<?php 
        echo $att;
        ?>
);
        output.SwapColumn (col_<?php 
        echo $att;
        ?>
, <?php 
        echo attSlot($att);
        ?>
);
<?php 
    }
    /*foreach*/
    ?>
    }

    // put in the output bitmap
    myOutBStringIter.Done ();
    output.SwapBitmap (myOutBStringIter);

    // and give back the result
    ChunkContainer tempResult (output);
    tempResult.swap (result);

    PROFILING2_END;

    PROFILING(start_time, "<?php 
    echo $wpName;
    ?>
", "LHS_lookup", "%d", totalNum);
    PROFILING(0.0, "HashTable", "fillrate", "%2.4f", HashTableSegment::globalFillRate*100.0);

    // Finish performance counters
    // Use the Set functionality in case we add additional counters later.
    PCounterList counterList;
    PCounter totalCnt("tpi lhs", totalNum, "<?php 
    echo $wpName;
    ?>
");
    counterList.Append(totalCnt);

    PROFILING2_SET(counterList, "<?php 
    echo $wpName;
    ?>
");

    int64_t hFillRate = int64_t(HashTableSegment::globalFillRate * 1000);
    PROFILING2_INSTANT("hfr", hFillRate, "global");

    free (serializeHere);
    return 1;
}
//+{"kind":"WPF", "name":"LHS Lookup", "action":"end"}

<?php 
}
Esempio n. 3
0
function JoinMerge($wpName, $jDesc)
{
    ?>


#define USE_PQ

#ifdef USE_PQ
//#include <algorithm>
#include <vector>
#else
#include <vector>
#include<set>
#endif

#include "WorkDescription.h"
#include "ExecEngineData.h"
#include "Column.h"
#include "ColumnIterator.cc"
#include "MMappedStorage.h"
#include "BString.h"
#include "BStringIterator.h"
#include "HashTableMacros.h"

#include <string.h>

using namespace std;

#define MAX_HASH 0xffffffffffff
#define MAX_NUM_CHUNKS 128 // maximum number of chunks we can have for linear scan solution

#ifndef USE_PQ
struct HashWrapper {
    HashWrapper() : hash(MAX_HASH), chunkNo(0) {}
    __uint64_t hash:48;
    __uint64_t chunkNo:16;
};

#endif

<?php 
    // Macro to find minimum value in a list
    // this is the loop that finds the best entry using linear scan
    // # minIndex$1 is set at the best position
    // in put assumed in hashes$1
    // Arguments:
    // $side is "LHS" or "RHS"
    $FIND_MIN = function ($side) {
        ?>
    best<?php 
        echo $side;
        ?>
= MAX_HASH;
    for (int i=0; i < num<?php 
        echo $side;
        ?>
Chunks; i++) {
        HT_INDEX_TYPE hash = hashes<?php 
        echo $side;
        ?>
[i]; // current value
        minIndex<?php 
        echo $side;
        ?>
 = (best<?php 
        echo $side;
        ?>
 > hash) ? i : minIndex<?php 
        echo $side;
        ?>
;
        best<?php 
        echo $side;
        ?>
 = (best<?php 
        echo $side;
        ?>
 > hash) ? hash: best<?php 
        echo $side;
        ?>
;
    }
    // remove this after debugging
    FATALIF(best<?php 
        echo $side;
        ?>
 != MAX_HASH && WHICH_SEGMENT(best<?php 
        echo $side;
        ?>
) !=0,
    "Hash out of range %llu", (unsigned long long) best<?php 
        echo $side;
        ?>
);
<?php 
    };
    ?>

/*
void check_correctness(int start, int end, BStringIterator& biter, ColumnIterator<__uint64_t>& hiter) {
    assert(start == end);
    int s = 0;
    int e = 262143;
    if (start > 0) {
        s = start * 262143 + (start-1);
        e = s + 262143 + 1;
    }
    int tuple = 0;
    while (!biter.AtEndOfColumn()) {
        if (!(s < hiter.GetCurrent() && hiter.GetCurrent() <= e))
            printf("\n -------   (%d,%d)(%d,%d)  currVal = %d", start,end,s,e,hiter.GetCurrent());
            assert(s <= hiter.GetCurrent() && hiter.GetCurrent() <= e);
            hiter.Advance();
            biter.Advance();
            tuple++;
        }
        printf("\n num tuples = %d", tuple);
    }
*/

#ifndef USE_PQ
// make heap comparator
struct compare_key {
    bool operator()( const HashWrapper lhs, const HashWrapper rhs )
    {
        return (rhs.hash < lhs.hash);
    }
};

struct compare_key_rev {
    bool operator()( const HashWrapper lhs, const HashWrapper rhs )
    {
        return (lhs.hash < rhs.hash);
    }
};
#endif


<?php 
    // More macros
    // Macro to Advance() columns
    // $side is LHS or RHS
    // $chunkNum is the chunk number
    // $sde is Lhs or Rhs
    $ADVANCE_CALL = function ($side, $chunkNum, $sde) use($jDesc) {
        $list = "attribute_queries_" . $side;
        foreach ($jDesc->{$list} as $att => $queries) {
            ?>
    col<?php 
            echo $side;
            ?>
IterVec_<?php 
            echo $att;
            ?>
[<?php 
            echo $chunkNum;
            ?>
].Advance();
    <?php 
        }
        ?>
    myInBStringIter<?php 
        echo $sde;
        ?>
Vec[<?php 
        echo $chunkNum;
        ?>
].Advance();
    col<?php 
        echo $side;
        ?>
IterVecHash[<?php 
        echo $chunkNum;
        ?>
].Advance();
<?php 
    };
    $ADVANCE_SEARCH_CALL = function ($col, $index, $side, $sde) use($jDesc, $FIND_MIN) {
        $list = "attribute_queries_" . $side;
        ?>
    //while (!myInBStringIter$4Vec[$2].AtEndOfColumn())
    if (!myInBStringIter<?php 
        echo $sde;
        ?>
Vec[<?php 
        echo $index;
        ?>
].AtEndOfColumn()) {
    //while (!<?php 
        echo $col;
        ?>
[<?php 
        echo $index;
        ?>
].AtUnwrittenByte())
        // Now find the first tuple for which query is active
        //Bitstring curBits(myInBStringIterLhsVec[<?php 
        echo $index;
        ?>
].GetCurrent ());
        //curBits.Intersect (queriesToRun);
        // If tuple has some active query, fill the heap
        //if (!curBits.IsEmpty())
        totalguys<?php 
        echo $side;
        ?>
++;
#ifdef USE_PQ
        // set new value in place of old value >
        // ALIN: Please fix the line below, I don't know what the $1 is supposed to be '
        hashes<?php 
        echo $side;
        ?>
[minIndex<?php 
        echo $side;
        ?>
]  = <?php 
        echo $col;
        ?>
[minIndex<?php 
        echo $side;
        ?>
].GetCurrent();
#else
        HashWrapper w;
        w.hash = <?php 
        echo $col;
        ?>
[<?php 
        echo $index;
        ?>
].GetCurrent(); // assume first column of each chunk is hash
        w.chunkNo = <?php 
        echo $index;
        ?>
;

        minHeap<?php 
        echo $side;
        ?>
.insert (w);
#endif
/*
        break; // while loop
    } else {
        // If this tuple don't have active query, advance all the columns of this chunk '
        // including Bitstring column
<?php 
        foreach ($jDesc->{$list} as $att) {
            ?>
        col<?php 
            echo $side;
            ?>
IterVec_<?php 
            echo $att;
            ?>
[<?php 
            echo $index;
            ?>
].Advance();
<?php 
        }
        /* foreach */
        ?>
        myInBStringIter<?php 
        echo $sde;
        ?>
Vec[<?php 
        echo $index;
        ?>
].Advance();
        col<?php 
        echo $side;
        ?>
IterVecHash[<?php 
        echo $index;
        ?>
].Advance();
    }
*/
    }
#ifdef USE_PQ
    <?php 
        $FIND_MIN($side);
        ?>
#endif
<?php 
    };
    /* ADVANCE_SEARCH_CALL */
    ?>

/*
    This takes two sorted list of chunks, lhs list and rhs list, and do the sort merge join.
    It maintaines a min heap of hash values on top of each list and virtually now we have
    just 2 lists to do sort merge. When some value matches from LHS and RHS heap, we need
    to checkpoint all the iterators of all the columns of all the RHS chunks including storing
    the heap, so that we can restore them all if another consecutive LHS value matches.
*/

//+{"kind":"WPF", "name":"Merge", "action":"start"}
extern "C"
int JoinMergeWorkFunc_<?php 
    echo $wpName;
    ?>
_writer (WorkDescription &workDescription, ExecEngineData &result) {

int totalguysLHS = 0;
int totalguysRHS = 0;
int total = 0;
int totalhash = 0;

    double start_time = global_clock.GetTime();

    // get the input LHS and RHS chunk container from work descripton
    JoinMergeWorkDescription myWork;
    myWork.swap (workDescription);
    ContainerOfChunks &inputLHSList = myWork.get_chunksLHS ();
    ContainerOfChunks &inputRHSList = myWork.get_chunksRHS ();

    int start = myWork.get_start();
    int end = myWork.get_end();

    // get the number of chunks in each list to create vectors of this length
    int numLHSChunks = inputLHSList.Length();
    int numRHSChunks = inputRHSList.Length();

    // get the waypoint identifier
    unsigned int wayPointID = myWork.get_wayPointID ().GetID();

    vector<ColumnIterator<__uint64_t> > colLHSIterVecHash;
    vector<ColumnIterator<__uint64_t> > colRHSIterVecHash;
    colLHSIterVecHash.resize(numLHSChunks);
    colRHSIterVecHash.resize(numRHSChunks);

    // get the input bitmap out of the LHS input chunks
    vector<BStringIterator> myInBStringIterLhsVec;
    myInBStringIterLhsVec.resize(numLHSChunks); // set the vector size
    assert(inputLHSList.Length());
    inputLHSList.MoveToStart();
    int i = 0;
    while (inputLHSList.RightLength()) {
        BStringIterator myInBStringIter;
        inputLHSList.Current().SwapBitmap (myInBStringIter);
        myInBStringIterLhsVec[i].swap (myInBStringIter);
        myInBStringIterLhsVec[i].SetFragmentRange(start, end);

        // get the hash now
        Column col_hash;
        inputLHSList.Current().SwapHash(col_hash);
        assert(col_hash.IsValid());
        ColumnIterator<__uint64_t> iter (col_hash, start, end);
        colLHSIterVecHash[i].swap(iter);
        //check_correctness(start, end, myInBStringIterLhsVec[i], colLHSIterVecHash[i]);
        inputLHSList.Advance ();
        i++;
    }

    // get the input bitmap out of the RHS input chunks
    vector<BStringIterator> myInBStringIterRhsVec;
    myInBStringIterRhsVec.resize(numRHSChunks); // set the vector size
    assert(inputRHSList.Length());
    inputRHSList.MoveToStart();
    i = 0;
    while (inputRHSList.RightLength()) {
        BStringIterator myInBStringIter;
        inputRHSList.Current().SwapBitmap(myInBStringIter);
        myInBStringIterRhsVec[i].swap (myInBStringIter);
        myInBStringIterRhsVec[i].SetFragmentRange(start, end);
        //fprintf(stderr, "\nRHS %d", myInBStringIterRhsVec[i].GetNumTuples());
        // get the hash now
        Column col_hash;
        inputRHSList.Current().SwapHash(col_hash);
        assert(col_hash.IsValid());
        ColumnIterator<__uint64_t> iter (col_hash, start, end);
        colRHSIterVecHash[i].swap(iter);
        //check_correctness(start, end, myInBStringIterRhsVec[i], colRHSIterVecHash[i]);
        inputRHSList.Advance ();
        i++;
    }

    // get all of the queries that are active here
    QueryIDSet queriesToRun = QueryExitsToQueries(myWork.get_whichQueryExits ());

    // this is the output chunk
    Chunk output;

    // create output iterators
<?php 
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $queries) {
        ?>
    MMappedStorage store_<?php 
        echo $att;
        ?>
;
    Column  col_<?php 
        echo $att;
        ?>
(store_<?php 
        echo $att;
        ?>
);
    <?php 
        echo attIteratorType($att);
        ?>
  colLHSOutIter_<?php 
        echo $att;
        ?>
(col_<?php 
        echo $att;
        ?>
);
<?php 
    }
    /*foreach*/
    ?>

<?php 
    foreach ($jDesc->attribute_queries_RHS_copy as $att => $queries) {
        ?>
    MMappedStorage store_<?php 
        echo $att;
        ?>
;
    Column  col_<?php 
        echo $att;
        ?>
(store_<?php 
        echo $att;
        ?>
);
    <?php 
        echo attIteratorType($att);
        ?>
  colRHSOutIter_<?php 
        echo $att;
        ?>
(col_<?php 
        echo $att;
        ?>
);
<?php 
    }
    /*foreach*/
    ?>

    // Create output BitString
    MMappedStorage myStore;
    Column bitmapOut (myStore);
    BStringIterator myOutBStringIter (bitmapOut, queriesToRun);

    // Build input iterators vectors first
    // Define only those attributes iterator which are required by some query
    // vector contains same type of column in all chunks, each vector index represents
    // a chunk
<?php 
    foreach ($jDesc->attribute_queries_LHS as $att => $queries) {
        ?>
    vector< <?php 
        echo attIteratorType($att);
        ?>
 > colLHSIterVec_<?php 
        echo $att;
        ?>
;
    colLHSIterVec_<?php 
        echo $att;
        ?>
.resize(numLHSChunks);
<?php 
    }
    /* foreach */
    ?>

<?php 
    foreach ($jDesc->attribute_queries_RHS as $att => $queries) {
        ?>
    vector< <?php 
        echo attIteratorType($att);
        ?>
 > colRHSIterVec_<?php 
        echo $att;
        ?>
;
    colRHSIterVec_<?php 
        echo $att;
        ?>
.resize(numRHSChunks);
<?php 
    }
    /* foreach */
    ?>

    // Extract columns now.
    // This extracts columns if there is any query for it in this WP
    i = 0;
<?php 
    foreach ($jDesc->attribute_queries_LHS as $att => $queries) {
        ?>
    {
        i = 0;
        inputLHSList.MoveToStart();
        while (inputLHSList.RightLength()) {
            // WP node queries intersect Translator provided queries
            QueryIDSet <?php 
        echo attQrys($att);
        ?>
(<?php 
        echo $queries;
        ?>
, true);
<?php 
        cgExtractColumnFragment($att, "inputLHSList.Current()", "start", "end", $wpName);
        ?>
            if (<?php 
        echo attQrys($att);
        ?>
.Overlaps(queriesToRun)){
                colLHSIterVec_<?php 
        echo $att;
        ?>
[i].swap(<?php 
        echo attData($att);
        ?>
);
            }
            inputLHSList.Advance ();
            i++;
        }
    }
<?php 
    }
    /* foreach */
    ?>

<?php 
    foreach ($jDesc->attribute_queries_RHS as $att => $queries) {
        ?>
    {
        i = 0;
        inputRHSList.MoveToStart();
        while (inputRHSList.RightLength()) {
            // WP node queries intersect Translator provided queries
            QueryIDSet <?php 
        echo attQrys($att);
        ?>
(<?php 
        echo $queries;
        ?>
, true);
<?php 
        cgExtractColumnFragment($att, "inputRHSList.Current()", "start", "end", $wpName);
        ?>
            if (<?php 
        echo attQrys($att);
        ?>
.Overlaps(queriesToRun)){
                colRHSIterVec_<?php 
        echo $att;
        ?>
[i].swap(<?php 
        echo attData($att);
        ?>
);
            }
            inputRHSList.Advance ();
            i++;
        }
    }
<?php 
    }
    /* foreach */
    ?>


    // Here we start the sort merge join
    // Create priority queue for hash column values
#ifdef USE_PQ
    HT_INDEX_TYPE hashesLHS[MAX_NUM_CHUNKS];
    HT_INDEX_TYPE hashesRHS[MAX_NUM_CHUNKS];
    FATALIF(numLHSChunks >= MAX_NUM_CHUNKS, "Too many chunks for LHS in Merge");
    FATALIF(numRHSChunks >= MAX_NUM_CHUNKS, "Too many chunks for RHS in Merge");
    HT_INDEX_TYPE bestLHS = MAX_HASH;
    HT_INDEX_TYPE bestRHS = MAX_HASH;
    int minIndexLHS = -1; // this indicates the chunk that is the best at this point
    // this invariant is maintained throught the code
    int minIndexRHS = -1;
#else
    multiset<HashWrapper, compare_key_rev> minHeapLHS;
    multiset<HashWrapper, compare_key_rev> minHeapRHS;
#endif

    // Fill the first value of each LHS chunk in LHS heap
    for (i = 0; i < numLHSChunks; i++) {
#ifdef USE_PQ
        minIndexLHS = i;
#endif
<?php 
    $ADVANCE_SEARCH_CALL("colLHSIterVecHash", "i", "LHS", "Lhs");
    ?>
    }

    // Fill the first value of each RHS chunk in RHS heap
    for (i = 0; i < numRHSChunks; i++) {
#ifdef USE_PQ
        minIndexRHS = i;
#endif
<?php 
    $ADVANCE_SEARCH_CALL("colRHSIterVecHash", "i", "RHS", "Rhs");
    ?>
    }

    // Now pick one of each heap and keep comparing until one of the heap is exhausted
#ifdef USE_PQ
/*    while (hashesLHS[minIndexLHS] != MAX_HASH && hashesRHS[minIndexRHS] != MAX_HASH) {
        HT_INDEX_TYPE wl = hashesLHS[minIndexLHS];
        HT_INDEX_TYPE wr = hashesRHS[minIndexRHS]; -- INEFFICIENT */

    while (bestLHS != MAX_HASH && bestRHS != MAX_HASH) {
        HT_INDEX_TYPE wl = bestLHS;
        HT_INDEX_TYPE wr = bestRHS;

#else
    while (!minHeapLHS.empty() && !minHeapRHS.empty()) {
        HashWrapper wlT = *(minHeapLHS.begin());
        HashWrapper wrT = *(minHeapRHS.begin());
        HT_INDEX_TYPE wl = wlT.hash;
        HT_INDEX_TYPE wr = wrT.hash;
        int minIndexLHS = wlT.chunkNo;
        int minIndexRHS = wrT.chunkNo;
#endif

        if (wl < wr) {
            //    printf("\n Hash val not found is = %ld, segment = %ld, slot = %ld", wl, WHICH_SEGMENT (wl), WHICH_SLOT (wl)); fflush(stdout); assert(0);

            // erase the minimum element
#ifdef USE_PQ
            hashesLHS[minIndexLHS] = MAX_HASH;
#else
            assert (!minHeapLHS.empty());
            minHeapLHS.erase(minHeapLHS.begin());
#endif

            // here advance all columns of LHS of chunk number wl.chunkNo
<?php 
    $ADVANCE_CALL("LHS", "minIndexLHS", "Lhs");
    $ADVANCE_SEARCH_CALL("colLHSIterVecHash", "minIndexLHS", "LHS", "Lhs");
    ?>

        } else if (wl > wr) {

            // erase the minimum element
#ifdef USE_PQ
            hashesRHS[minIndexRHS] = MAX_HASH;
#else
            assert (!minHeapRHS.empty());
            minHeapRHS.erase(minHeapRHS.begin());
#endif

            // here advance all columns of RHS of chunk number wr.chunkNo
<?php 
    $ADVANCE_CALL("RHS", "minIndexRHS", "Rhs");
    $ADVANCE_SEARCH_CALL("colRHSIterVecHash", "minIndexRHS", "RHS", "Rhs");
    ?>

        } else { // (wl == wr)

            HT_INDEX_TYPE matchingHash = wl;

            // Save checkpoint for all rhs columns before incrementing both chunks
            for (int chk = 0; chk < numRHSChunks; chk++) {
<?php 
    foreach ($jDesc->attribute_queries_RHS as $att => $queries) {
        ?>
                    colRHSIterVec_<?php 
        echo $att;
        ?>
[chk].CheckpointSave();
<?php 
    }
    /* foreach */
    ?>
                myInBStringIterRhsVec[chk].CheckpointSave();
                colRHSIterVecHash[chk].CheckpointSave();
            }

            // Also save the state of the heap
#ifdef USE_PQ
            HT_INDEX_TYPE hashesRHS_copy[MAX_NUM_CHUNKS];
            memcpy(hashesRHS_copy, hashesRHS, numRHSChunks*sizeof(HT_INDEX_TYPE));
            int minIndexRHS_copy = minIndexRHS;
#else
            multiset<HashWrapper, compare_key_rev> minHeapRHSCheckpoint(minHeapRHS);
#endif

#ifdef USE_PQ
            while (bestLHS != MAX_HASH)
#else
            while (!minHeapLHS.empty()) // break from this if 2 consecutive LHS mismatches
#endif
	    {
#ifdef USE_PQ
//                HT_INDEX_TYPE wl1 = hashesLHS[minIndexLHS];
                HT_INDEX_TYPE wl1 = bestLHS;
#else
                HashWrapper wl1T = *(minHeapLHS.begin());
                HT_INDEX_TYPE wl1 = wl1T.hash;
                int minIndexLHS = wl1T.chunkNo;
#endif

                if (wl1 != matchingHash) { // next LHS dont match to previous LHS hash, first time always match
                    break;

                } else { // restore everything

                    // restore the RHS Column iterators to original value
                    for (int chk = 0; chk < numRHSChunks; chk++) {
<?php 
    foreach ($jDesc->attribute_queries_RHS as $att => $queries) {
        ?>
                            colRHSIterVec_<?php 
        echo $att;
        ?>
[chk].CheckpointRestore();
<?php 
    }
    /* foreach */
    ?>
                            myInBStringIterRhsVec[chk].CheckpointRestore();
                            colRHSIterVecHash[chk].CheckpointRestore();
                    }
                    // restore the original heap state
#ifdef USE_PQ
                    memcpy(hashesRHS, hashesRHS_copy,  numRHSChunks*sizeof(HT_INDEX_TYPE));
                    minIndexRHS = minIndexRHS_copy;
                    bestRHS = hashesRHS[minIndexRHS_copy];
#else
                    minHeapRHS.clear();
                    minHeapRHS = minHeapRHSCheckpoint;
#endif
                }

#ifdef USE_PQ
//                while (hashesRHS[minIndexRHS] != MAX_HASH) {
                while (bestRHS != MAX_HASH) {
#else
                while (!minHeapRHS.empty()) {
#endif

#ifdef USE_PQ
//                    HT_INDEX_TYPE wr1 = hashesRHS[minIndexRHS];
                    HT_INDEX_TYPE wr1 = bestRHS;
#else
                    HashWrapper wr1T = *(minHeapRHS.begin());
                    HT_INDEX_TYPE wr1 = wr1T.hash;
                    int minIndexRHS = wr1T.chunkNo;
#endif

                    if (wl1 == wr1) { // first one will obviously match as it matched before
                        // Merge all columns here for wl1.chunkNo and wr1.chunkNo after matching attributes

                        // Make sure both of their bitstrings intersect
                        Bitstring rez = queriesToRun;
                        rez.Intersect (myInBStringIterRhsVec[minIndexRHS].GetCurrent ());
                        rez.Intersect (myInBStringIterLhsVec[minIndexLHS].GetCurrent ());

                        // This contains union of all translater queries
                        Bitstring uni = 0;
                        Bitstring qBits;
                        totalhash++;


                        // Do the actual comparision
                        bool anyOneMatch = false;
<?php 
    foreach ($jDesc->queries_attribute_comparison as $qClass) {
        ?>
                            // See if any query in query class is eligible for this comparision
                            qBits = QueryIDSet(<?php 
        echo $qClass->qClass;
        ?>
, true);
                            qBits.Intersect(rez);
                            if (
                            /*    !qBits.IsEmpty () && */
    <?php 
        foreach ($qClass->att_pairs as $pair) {
            ?>
                                colLHSIterVec_<?php 
            echo $pair->lhs;
            ?>
[minIndexLHS].GetCurrent() == colRHSIterVec_<?php 
            echo $pair->rhs;
            ?>
[minIndexRHS].GetCurrent() &&
    <?php 
        }
        /*foreach pair*/
        ?>
                                1 ) {
                                anyOneMatch = true;
                                uni.Union (qBits);
                            }
<?php 
    }
    /* foreach qClass */
    ?>

                        if (anyOneMatch) {
                            // fill the output iterators
<?php 
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $queries) {
        ?>
                                //if (rez.Overlaps(<?php 
        echo $queries;
        ?>
)) { Should we check translator queries?
                                    colLHSOutIter_<?php 
        echo $att;
        ?>
.Insert(colLHSIterVec_<?php 
        echo $att;
        ?>
[minIndexLHS].GetCurrent());
                                    colLHSOutIter_<?php 
        echo $att;
        ?>
.Advance();
                                //}
<?php 
    }
    /* foreach */
    ?>


<?php 
    foreach ($jDesc->attribute_queries_RHS_copy as $att => $queries) {
        ?>
                                //if (rez.Overlaps(<?php 
        echo $queries;
        ?>
)) { Should we check translator queries?
                                    colRHSOutIter_<?php 
        echo $att;
        ?>
.Insert(colRHSIterVec_<?php 
        echo $att;
        ?>
[minIndexRHS].GetCurrent());
                                    colRHSOutIter_<?php 
        echo $att;
        ?>
.Advance();
                                //}
<?php 
    }
    /* foreach */
    ?>

                            myOutBStringIter.Insert (uni);
                            total++;
                            myOutBStringIter.Advance ();
                        } else {
                            //Bitstring b(0, true);
                            //myOutBStringIter.Insert (b);
                            //myOutBStringIter.Advance ();
                        }

                        // erase the minimum element
#ifdef USE_PQ
                        hashesRHS[minIndexRHS] = MAX_HASH;
#else
                        minHeapRHS.erase(minHeapRHS.begin());
#endif

                        // here advance all columns of RHS of chunk number wr1.chunkNo
<?php 
    $ADVANCE_CALL("RHS", "minIndexRHS", "Rhs");
    $ADVANCE_SEARCH_CALL("colRHSIterVecHash", "minIndexRHS", "RHS", "Rhs");
    ?>
                    } else {
                        break;
                    }
                }

                // erase the minimum element
#ifdef USE_PQ
                hashesLHS[minIndexLHS] = MAX_HASH;
#else
                minHeapLHS.erase(minHeapLHS.begin());
#endif

                // here advance all columns of LHS of chunk number wl1.chunkNo
<?php 
    $ADVANCE_CALL("LHS", "minIndexLHS", "Lhs");
    $ADVANCE_SEARCH_CALL("colLHSIterVecHash", "minIndexLHS", "LHS", "Lhs");
    ?>
            }
        }
    }

    // fill the output iterators
<?php 
    foreach ($jDesc->attribute_queries_LHS_copy as $att => $queries) {
        ?>
    Column collhs_<?php 
        echo $att;
        ?>
;
    colLHSOutIter_<?php 
        echo $att;
        ?>
.Done(collhs_<?php 
        echo $att;
        ?>
);
    output.SwapColumn (collhs_<?php 
        echo $att;
        ?>
, <?php 
        echo attSlot($att);
        ?>
);
<?php 
    }
    /* foreach */
    foreach ($jDesc->attribute_queries_RHS_copy as $att => $queries) {
        ?>
    Column colrhs_<?php 
        echo $att;
        ?>
;
    colRHSOutIter_<?php 
        echo $att;
        ?>
.Done(colrhs_<?php 
        echo $att;
        ?>
);
    output.SwapColumn (colrhs_<?php 
        echo $att;
        ?>
, <?php 
        echo attSlot($att);
        ?>
);
<?php 
    }
    /* foreach */
    ?>

    //myOutBStringIter.Done (bitmapOut);
    myOutBStringIter.Done ();
//    printf("\nTuples %d %d %d %d %d", myOutBStringIter.GetNumTuples(), total, totalguysLHS, totalguysRHS, totalhash);
//    fflush(stdout);

    PROFILING(start_time, "<?php 
    echo $wpName;
    ?>
", "Merge", "%d\t%d", totalhash, total);

    //output.SwapBitmap (bitmapOut);
    output.SwapBitmap (myOutBStringIter);
/*
    MMappedStorage st;
    Column co (st);
    Bitstring patt(0xf, true);
    BStringIterator iter(co, patt, 0);
    output.SwapBitmap(iter);
*/
    // and give back the result
    ChunkContainer tempResult (output);
    tempResult.swap (result);

    return 0; // have to return something
}
//+{"kind":"WPF", "name":"Merge", "action":"end"}

<?php 
}