// // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // require_once 'DataFunctions.php'; ?> #ifndef _GI_INTERNAL_H_ #define _GI_INTERNAL_H_ #include "ContainerTypes.h" #include "GLAData.h" #include "GIStreamInfo.h" #include "TwoWayList.h" <?php grokit\create_data_type("GITask", "Data", [], ['stream' => 'GIStreamProxy', 'gi' => 'GLAState']); ?> typedef TwoWayList<GITask> GITaskList; #endif // _GI_INTERNAL_H_
/** These are requests going to the disk. The pageSize is the MMap page The requests are sent to the DiskArray who sends them to the HDThreads. Arguments: memLoc: the memory location where the read/write is done */ <?php grokit\create_data_type("DiskRequestData", "DiskData", ['memLoc' => 'void*'], []); ?> typedef TwoWayList< DiskRequestData > DiskRequestDataContainer; /** Internal datatype used by the ChunkReaderWriter to keep track of requests Arguments: chunkID: id of the chunk we are dealing with hMsg: the hopping message to send back to EE token: the token to send back */ <?php grokit\create_data_type("CRWRequest", "Data", ['chunkID' => 'off_t'], ['hMsg' => 'HoppingDataMsg', 'token' => 'GenericWorkToken']); ?> typedef Keyify<off_t> KOff_t; // keyified off_t typedef EfficientMap< KOff_t, CRWRequest > IDToRequestMap; #endif // DISK_IO_DATA_
// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // require_once 'DataFunctions.php'; ?> #ifndef _BULK_LOADER_INTERNAL_H_ #define _BULK_LOADER_INTERNAL_H_ #include "ContainerTypes.h" /* Data types used by Bulk Loader to do its internal work. Should be of no interest to the rest of the system */ <?php grokit\create_data_type("TextLoaderDS", "Data", ['stream' => 'FILE*', 'file' => 'std::string'], []); ?> // container for the above typedef TwoWayList<TextLoaderDS> TextLoaderDSContainer; #endif // _BULK_LOADER_INTERNAL_H_
<?php grokit\create_data_type("GISTProduceResultsWorkFunc", "WorkFuncWrapper", [], [], true); ?> <?php grokit\create_data_type("GISTProduceStateWorkFunc", "WorkFuncWrapper", [], [], true); ?> <?php grokit\create_data_type("CacheChunkWorkFunc", "WorkFuncWrapper", [], [], true); ?> <?php grokit\create_data_type("GSEPreProcessWorkFunc", "WorkFuncWrapper", [], [], true); ?> <?php grokit\create_data_type("GSEProcessReadOnlyWorkFunc", "WorkFuncWrapper", [], [], true); ?> <?php grokit\create_data_type("ClusterProcessChunkWorkFunc", "WorkFuncWrapper", [], [], true); ?> <?php grokit\generate_deserializer('WorkFuncWrapper'); ?> #endif
// Produces results for this GIST. <?php grokit\create_data_type("GISTProduceResultsWD", "WorkDescription", ['fragmentNo' => 'int'], ['whichOne' => 'QueryExit', 'gist' => 'GLAState']); ?> <?php grokit\create_data_type("GIProduceChunkWD", "WorkDescription", [], ['gi' => 'GLAState', 'stream_info' => 'GIStreamProxy', 'queriesCovered' => 'QueryIDSet']); ?> /***** GSE Work Descriptions *****/ // Pre-processing <?php grokit\create_data_type("GSEPreProcessWD", "WorkDescription", [], ['whichQueryExits' => 'QueryExitContainer', 'requiredStates' => 'QueryToGLASContMap']); ?> // Processing a read-only request <?php grokit\create_data_type("GSEProcessWD", "WorkDescription", [], ['whichQueryExits' => 'QueryExitContainer', 'gseStates' => 'QueryToGLAStateMap', 'constStates' => 'QueryToGLAStateMap', 'request' => 'ServiceData']); ?> /****** Cluster Work Descriptions ******/ // Process Chunk <?php grokit\create_data_type('ClusterProcessChunkWD', 'WorkDescription', [], ['chunk' => 'Chunk']); ?> #endif // WORK_DESCRIPTION_H
// you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // require_once 'DataFunctions.php'; ?> #include <inttypes.h> #include <string> #include "Data.h" #include "TwoWayList.h" /** Container for counters */ <?php grokit\create_data_type("PCounter", "DataC", ['name' => 'std::string', 'value' => 'int64_t', 'group' => 'std::string'], [], true); ?> typedef TwoWayList <PCounter> PCounterList; #endif // _PROF_MSG_DATA_H_
/** Text Loader Arguments: files: names of files that we are bulkloading queries: the queries that are going to tag the produced chunks */ <?php grokit\create_data_type("TextLoaderConfigureData", "WayPointConfigureData", ['files' => 'StringContainer'], ['queries' => 'QueryExitContainer'], true); ?> <?php grokit\create_data_type("TileJoinConfigureData", "WayPointConfigureData", ['lhsrelName' => 'std::string', 'rhsrelName' => 'std::string'], ['JoinWP' => 'WayPointID', 'queryColumnsMapLhs' => 'QueryExitToSlotsMap', 'queryColumnsMapRhs' => 'QueryExitToSlotsMap', 'columnsToSlotsPairLhs' => 'SlotPairContainer', 'columnsToSlotsPairRhs' => 'SlotPairContainer'], true); ?> <?php grokit\create_data_type("GIConfigureData", "WayPointConfigureData", ['files' => 'StringContainer'], ['queries' => 'QueryExitContainer'], true); ?> <?php grokit\generate_deserializer('WayPointConfigureData'); ?> // this is the list of way point configurations sent into the execution engine typedef TwoWayList<WayPointConfigureData> WayPointConfigurationList; #endif
// Copyright 2013 Tera Insights, LLC. All Rights Reserved. require_once 'DataFunctions.php'; ?> #ifndef _EXEC_ENGINE_TASKS_H_ #define _EXEC_ENGINE_TASKS_H_ #include <cinttypes> #include <string> #include "TwoWayList.h" // Base class for Execution Engine Tasks <?php grokit\create_base_data_type("Task", "DataC", [], [], true, true); ?> typedef TwoWayList<Task> TaskList; // Delete Relation <?php grokit\create_data_type("DeleteRelationTask", "Task", ['relation' => 'std::string'], [], true); ?> <?php grokit\generate_deserializer('Task'); ?> #endif // _EXEC_ENGINE_TASKS_H_
?> // ProduceResults work function will produce a ChunkContainer // Result of a GI reading data from a file. <?php grokit\create_data_type("GIProduceChunkRez", "ExecEngineData", [], ['stream' => 'GIStreamProxy', 'gi' => 'GLAState', 'chunk' => 'ChunkContainer']); ?> /***** Return types for GSEs *****/ // Results from preprocessing <?php grokit\create_data_type("GSEPreProcessRez", "ExecEngineData", [], ['constStates' => 'QueryToGLAStateMap']); ?> // Results from processing <?php grokit\create_data_type("GSEProcessRez", "ExecEngineData", [], ['gseStates' => 'QueryToGLAStateMap', 'result' => 'ServiceData']); ?> /***** Return types for Cluster WP ******/ <?php grokit\create_data_type("ClusterProcessChunkRez", "ExecEngineData", ['min' => 'int64_t', 'max' => 'int64_t'], []); ?> #endif
typedef TwoWayList <GLAState> GLAStateContainer; typedef EfficientMap <QueryID, GLAState> QueryToGLAStateMap; typedef EfficientMap <QueryID, GLAStateContainer> QueryToGLASContMap; typedef EfficientMap< QueryID, Swapify<int> > QueryIDToInt; typedef EfficientMap< QueryID, Swapify<bool> > QueryIDToBool; typedef TwoWayList<WayPointID> ReqStateList; typedef EfficientMap<QueryID, ReqStateList> QueryToReqStates; /** GLAPointer stores a pointer to a GLA state. This is an in-memory pointer. */ <?php grokit\create_data_type("GLAPtr", "GLAState", ['glaPtr' => 'void*'], [], false); ?> /******* Data containing QueryIDSet and Chunk **/ <?php grokit\create_base_data_type("CacheData", "DataC", ['queryIDs' => 'QueryIDSet'], ['cacheChunk' => 'Chunk'], false); ?> typedef EfficientMap <ChunkID, CacheData> ChunkToCacheMap; /** Data Structures for GIST Waypoints */ <?php grokit\create_base_data_type("GISTWorkUnit", "DataC", [], ['gist' => 'GLAState', 'localScheduler' => 'GLAState', 'gla' => 'GLAState'], false);
// needs to go, lineage is a list of History objects (where each waypoint that // obtains and then forwards on the message can add new history objects to the // list), and data is the actual data in the message. <?php grokit\create_base_data_type("HoppingDataMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExitContainer', 'lineage' => 'HistoryList', 'data' => 'ExecEngineData']); ?> // this macro defines direct messages, that are sent to a particular waypoint <?php grokit\create_base_data_type("DirectMsg", "Data", ['receiver' => 'WayPointID'], ['message' => 'Notification']); ?> // this macro defines hopping downstream messages... these are a lot like hopping // data messages, except that they contain Notification objects instead of data <?php grokit\create_data_type("HoppingDownstreamMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExitContainer', 'msg' => 'Notification']); ?> // this macro defines hopping upstream messages... unlike the downstream messages, // these have only one destination---the waypoint(s) that produce(s) data associated // with a particular query exit <?php grokit\create_data_type("HoppingUpstreamMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExit', 'msg' => 'Notification']); ?> #endif
// set of sampled entries from all of the too-full segments <?php grokit\create_data_type("TooFullMessage", "Notification", [], ['whatWeFound' => 'HashSegmentSample']); ?> // this is sent from the hash cleaner to a particular waypoint to notify the waypoint // that it has now become disk-based <?php grokit\create_data_type("IKillYouMessage", "Notification", [], []); ?> // when a join waypoint finlly dies (that is, when a "wounded" waypoint that is being emptied // out of the hash table has its last bytes ever removed from the hash table) it sends this // message to both the hash cleaner and to the writer letting them know that it has finally died <?php grokit\create_data_type("WayPointDeadMsg", "Notification", [], []); ?> // messge sent from cleaner to a join with the hash <?php grokit\create_data_type("CentralHashMessage", "Notification", [], ['centralHash' => 'HashTable']); ?> <?php grokit\create_data_type("GetHashTable", "Notification", [], []); ?> #endif
// this file contains all of the various work token types #include "Data.h" #include "DistributedQueue.cc" // this macro defines the generic token type, which is sent to waypoints when they // request the ability to do some work... tokens must also be sent back to the // execution engine when the work is done. grantedTo is the identifier for the // waypoint who was actually granted the right to use the token <?php grokit\create_base_data_type("GenericWorkToken", "Data", ['label' => 'int'], []); ?> // this macro defines the CPU token type <?php grokit\create_data_type("CPUWorkToken", "GenericWorkToken", [], []); ?> // this macro defines the Disk token type <?php grokit\create_data_type("DiskWorkToken", "GenericWorkToken", [], []); ?> // this is used to send a list of disk tokens for writer waypoints to use typedef DistributedQueue <DiskWorkToken> DiskWorkTokenQueue; #endif
<?php grokit\create_data_type("TextLoaderHistory", "History", ['whichChunk' => 'ChunkID', 'file' => 'std::string'], []); ?> <?php grokit\create_data_type("GIHistory", "History", ['whichChunk' => 'ChunkID', 'file' => 'std::string'], []); ?> <?php grokit\create_data_type("TileJoinScanHistory", "History", ['bucketID' => 'int', 'whichChunk' => 'ChunkID', 'isLHS' => 'bool', 'file' => 'std::string'], ['whichExits' => 'QueryExitContainer']); ?> <?php grokit\create_data_type("TileJoinWriteHistory", "History", [], []); ?> <?php grokit\create_data_type("TileJoinMergeHistory", "History", ['bucketID' => '__uint64_t'], ['chunkIDLHSList' => 'ChunkIDContainer', 'chunkIDRHSList' => 'ChunkIDContainer', 'whichExits' => 'QueryExitContainer']); ?> <?php grokit\create_data_type("CacheHistory", "History", ['whichChunk' => 'uint64_t'], []); ?> #endif