#include "JoinWayPointID.h"
#include "ExecEngineData.h"
#include "GIStreamInfo.h"
#include "ServiceData.h"
#include "DistributedCounter.h"
// this is the base class for the hierarchy of types that one can send to a CPU
// worker to describe the task that the worker is supposed to complete. In more
// detail, a CPU worker will always execute a function of type WorkFunc
// (see WorkerMessages.h.m4)... functions of type WorkFunc accept data of type
// WorkDescription. Sooo, if a specific waypoint wants a function executed, it
// sends a specific WorkFunc to a worker, along with a specific WorkDescription
// object that it has created to parameterize the function
<?php
grokit\create_base_data_type("WorkDescription", "Data", [], []);
?>
<?php
grokit\create_data_type("CacheChunkWD", "WorkDescription", [], ['chunkToProcess' => 'ChunkContainer']);
?>
<?php
grokit\create_data_type("SelectionPreProcessWD", "WorkDescription", [], ['whichQueryExits' => 'QueryExitContainer', 'requiredStates' => 'QueryToGLASContMap']);
?>
<?php
grokit\create_data_type("SelectionProcessChunkWD", "WorkDescription", ['chunkID' => 'ChunkID'], ['whichQueryExits' => 'QueryExitContainer', 'chunkToProcess' => 'Chunk', 'constStates' => 'QueryToGLAStateMap']);
?>
#define DISK_IO_DATA_
#include "Data.h"
#include "TwoWayList.h"
#include "Column.h"
#include "EEExternMessages.h"
// the base class of the hierarchy
/** The common parts are:
startPage: the starting page on the disk where data is read/written
sizePages: the size of the column in pages
*/
<?php
grokit\create_base_data_type("DiskData", "Data", ['startPage' => 'off_t', 'sizePages' => 'off_t'], []);
?>
/** Used by the ChunkReaderWriter to get a description of
the column content to write on the disk.
Arguments:
slot: which column is this about
sizePages: size in number of pages
startPage: the start of the column on disk
Sizes in bytes:
sizeCompressed: if !=0, the column is compressed
<?php
grokit\create_data_type("JoinMergeConfigureData", "WayPointConfigureData", [], [], true);
?>
<?php
grokit\create_data_type("CacheChunkConfigureData", "WayPointConfigureData", [], [], true);
?>
<?php
grokit\create_data_type("ClusterConfigureData", "WayPointConfigureData", ['relation' => 'std::string'], [], true);
?>
// General Processing WayPoint configuration data.
<?php
grokit\create_base_data_type("GPWConfigureData", "WayPointConfigureData", [], ['reqStates' => 'QueryToReqStates'], true);
?>
<?php
grokit\create_data_type("SelectionConfigureData", "GPWConfigureData", [], [], true);
?>
<?php
grokit\create_data_type("GLAConfigureData", "GPWConfigureData", [], ['resultIsState' => 'QueryIDToBool'], true);
?>
<?php
grokit\create_data_type("GTConfigureData", "GPWConfigureData", [], [], true);
?>
void ToJson( const WorkFunc & src, Json::Value & dest ) {
dest = Json::Value((Json::Int64) src);
}
inline
void FromJson( const Json::Value & src, WorkFunc & dest ) {
dest = (WorkFunc) src.asInt64();
}
// this contains all of the work func types
// this is the basic wrapper for the work functions that are sent into waypoints
// at configuration time. All functions to be run are of type "WorkFunc";
// see the file "WorkerMessages.h.m4" in CPUWorkers/m4 for the definition of "WorkFunc"
<?php
grokit\create_base_data_type("WorkFuncWrapper", "DataC", ['myFunc' => 'WorkFunc'], [], true, true);
?>
// this is a list of work functions that are used to configure a waypoint
typedef TwoWayList <WorkFuncWrapper> WorkFuncContainer;
// We should have one of these for each of the types of work funcs in the system.
// It does not have anything in it above and beyond what the WorkFuncWrapper has.
// but the fact that we have one of these for each of the work functions means that
// we can put the work functions in a list of swappable objects, and do things like
// search the list for the "AggFinishUpWorkFunc" function, or the "AggWorkFunc"
// function
//Join
<?php
grokit\create_data_type("JoinLHSWorkFunc", "WorkFuncWrapper", [], [], true);
#include "GLAData.h"
#include "HashTableSegment.h"
#include "Tokens.h"
#include "History.h"
#include "GIStreamInfo.h"
#include "ServiceData.h"
#include <cstdint>
// this file has all of the data types that can be sent downstream thru the
// data path graph
// this macro defines the "ExecEngineData" class, which is a generic container
// for data that is sent downstream from one waypoint to another to be processed.
<?php
grokit\create_base_data_type("ExecEngineData", "DataC", [], []);
?>
// this is the most common type of EEData object: one that contains a single chunk
<?php
grokit\create_data_type("ChunkContainer", "ExecEngineData", [], ['myChunk' => 'Chunk']);
?>
// A datatype used for caching rejected chunks, while keeping track of its
// history.
<?php
grokit\create_data_type("CachedChunk", "DataC", [], ['myChunk' => 'ChunkContainer', 'lineage' => 'HistoryList', 'whichExits' => 'QueryExitContainer']);
?>
#include <cstdio>
// this stores data containers that relate to the central hash table
// when someone is adding data to the hash table, they use their first 100 or so
// probes to collect info on the state of the table; this ob ject stores that info
<?php
grokit\create_base_data_type("HashEntrySummary", "DataC", ['whichWayPoint' => 'JoinWayPointID', 'whichQueries' => 'QueryID'], []);
?>
// we will have one HashEntrySummary associated with each probe
typedef TwoWayList <HashEntrySummary> HashSegmentSample;
// this is just a linked list of ints
typedef Swapify <int> SwapifiedInt;
typedef TwoWayList <SwapifiedInt> IntList;
// this is used to store a chunk that has been extracted from the central hash table...
// whichWayPoint tells which waypoint all of the data in the chunk originally came from
// isLHS is nonzero if and only if this is a LHS chunk; otherwise, it is a RHS one
// whichSegment identifies the particular hash table segment that this chunk came from
<?php
grokit\create_base_data_type("ExtractedChunk", "DataC", ['isLHS' => 'int', 'whichWayPoint' => 'WayPointID'], ['myChunk' => 'Chunk']);
?>
typedef TwoWayList <ExtractedChunk> ExtractionList;
#endif
/** GLAPointer stores a pointer to a GLA state. This is an in-memory
pointer.
*/
<?php
grokit\create_data_type("GLAPtr", "GLAState", ['glaPtr' => 'void*'], [], false);
?>
/******* Data containing QueryIDSet and Chunk **/
<?php
grokit\create_base_data_type("CacheData", "DataC", ['queryIDs' => 'QueryIDSet'], ['cacheChunk' => 'Chunk'], false);
?>
typedef EfficientMap <ChunkID, CacheData> ChunkToCacheMap;
/** Data Structures for GIST Waypoints */
<?php
grokit\create_base_data_type("GISTWorkUnit", "DataC", [], ['gist' => 'GLAState', 'localScheduler' => 'GLAState', 'gla' => 'GLAState'], false);
?>
typedef TwoWayList <GISTWorkUnit> GistWUContainer;
typedef EfficientMap<QueryID, GISTWorkUnit> QueryToGistWorkUnit;
typedef EfficientMap<QueryID, GistWUContainer> QueryToGistWUContainer;
#endif // GLA_DATA_H
// Copyright 2013 Tera Insights, LLC. All Rights Reserved.
require_once 'DataFunctions.php';
?>
#ifndef _EXEC_ENGINE_TASKS_H_
#define _EXEC_ENGINE_TASKS_H_
#include <cinttypes>
#include <string>
#include "TwoWayList.h"
// Base class for Execution Engine Tasks
<?php
grokit\create_base_data_type("Task", "DataC", [], [], true, true);
?>
typedef TwoWayList<Task> TaskList;
// Delete Relation
<?php
grokit\create_data_type("DeleteRelationTask", "Task", ['relation' => 'std::string'], [], true);
?>
<?php
grokit\generate_deserializer('Task');
?>
#endif // _EXEC_ENGINE_TASKS_H_
// Copyright 2013 Tera insights, LLC. All Rights Reserved.
require_once 'DataFunctions.php';
?>
/** Data Structures for Service Waypoints */
#ifndef _SERVICE_DATA_H_
#define _SERVICE_DATA_H_
#include <string>
#include "SerializeJson.h"
<?php
grokit\create_base_data_type("ServiceData", "DataC", ['id' => 'std::string', 'service' => 'std::string', 'kind' => 'std::string', 'data' => 'Json::Value'], [], true);
?>
namespace ServiceErrors {
using type = Json::Int;
constexpr type NoSuchService = 100;
constexpr type NoSuchOperation = 101;
constexpr type OperationNotSupported = 200;
inline
ServiceData MakeError( ServiceData& initialRequest, type error, std::string message, Json::Value data = Json::Value(Json::nullValue) ) {
Json::Value errData(Json::objectValue);
errData["code"] = error;
errData["message"] = message;
// identifier of the waypoint who is sending the message, since the execution engine
// delivers the message to the next logical recipient in the graph after currentPos
// this macro defines the "hopping data message", which is used to send data
// through the network from a data producer. dest is all of the places that the msg
// needs to go, lineage is a list of History objects (where each waypoint that
// obtains and then forwards on the message can add new history objects to the
// list), and data is the actual data in the message.
<?php
grokit\create_base_data_type("HoppingDataMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExitContainer', 'lineage' => 'HistoryList', 'data' => 'ExecEngineData']);
?>
// this macro defines direct messages, that are sent to a particular waypoint
<?php
grokit\create_base_data_type("DirectMsg", "Data", ['receiver' => 'WayPointID'], ['message' => 'Notification']);
?>
// this macro defines hopping downstream messages... these are a lot like hopping
// data messages, except that they contain Notification objects instead of data
<?php
grokit\create_data_type("HoppingDownstreamMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExitContainer', 'msg' => 'Notification']);
?>
// this macro defines hopping upstream messages... unlike the downstream messages,
// these have only one destination---the waypoint(s) that produce(s) data associated
// with a particular query exit
<?php
grokit\create_data_type("HoppingUpstreamMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExit', 'msg' => 'Notification']);
#ifndef NOTIFY_H
#define NOTIFY_H
#include "Data.h"
#include "ID.h"
#include "ExecEngineData.h"
#include "History.h"
#include "HashTable.h"
// this macro defines the objects that are used to store notifications that are
// sent from one waypoint to another in the network (notifications are things
// like: "ready to start processing chunks for a query exit" or "done sending
// chunks for a query exit"... sender identifies the waypoint who sent the
// notification
<?php
grokit\create_base_data_type("Notification", "DataC", ['sender' => 'WayPointID'], []);
?>
// this is a specific type of notification that tells us that a query or queries are done,
// in the sense that no more data is going to be coming through the system
<?php
grokit\create_data_type("QueryDoneMsg", "Notification", [], ['whichOnes' => 'QueryExitContainer']);
?>
// this type of notification tells the receiver that it is time to start sending data
// to one particualr query exit
<?php
grokit\create_data_type("StartProducingMsg", "Notification", [], ['whichOne' => 'QueryExit']);
?>
#ifndef TOKENS_H
#define TOKENS_H
// this file contains all of the various work token types
#include "Data.h"
#include "DistributedQueue.cc"
// this macro defines the generic token type, which is sent to waypoints when they
// request the ability to do some work... tokens must also be sent back to the
// execution engine when the work is done. grantedTo is the identifier for the
// waypoint who was actually granted the right to use the token
<?php
grokit\create_base_data_type("GenericWorkToken", "Data", ['label' => 'int'], []);
?>
// this macro defines the CPU token type
<?php
grokit\create_data_type("CPUWorkToken", "GenericWorkToken", [], []);
?>
// this macro defines the Disk token type
<?php
grokit\create_data_type("DiskWorkToken", "GenericWorkToken", [], []);
?>
// history specific to the hash table cleaner
<?php
grokit\create_data_type("HashCleanerHistory", "History", ['whichSegment' => 'int'], []);
?>
// history specific to a table scan waypoint
<?php
grokit\create_data_type("TableScanHistory", "History", ['whichChunk' => 'int'], ['whichExits' => 'QueryExitContainer']);
?>
// history specific to a table waypoint
<?php
grokit\create_base_data_type("TableHistory", "History", ['whichChunk' => 'ChunkID'], ['whichExits' => 'QueryExitContainer']);
?>
// two types of specific histories for Table
<?php
grokit\create_data_type("TableReadHistory", "TableHistory", [], []);
?>
// this is not used now but might be used in the future
// if writer part of Table neds some extra data
<?php
grokit\create_data_type("TableWriteHistory", "TableHistory", [], []);
?>
