//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
require_once 'DataFunctions.php';
?>
#ifndef _GI_INTERNAL_H_
#define _GI_INTERNAL_H_
#include "ContainerTypes.h"
#include "GLAData.h"
#include "GIStreamInfo.h"
#include "TwoWayList.h"
<?php
grokit\create_data_type("GITask", "Data", [], ['stream' => 'GIStreamProxy', 'gi' => 'GLAState']);
?>
typedef TwoWayList<GITask> GITaskList;
#endif // _GI_INTERNAL_H_
/** These are requests going to the disk. The pageSize is the MMap page
The requests are sent to the DiskArray who sends them to the HDThreads.
Arguments:
memLoc: the memory location where the read/write is done
*/
<?php
grokit\create_data_type("DiskRequestData", "DiskData", ['memLoc' => 'void*'], []);
?>
typedef TwoWayList< DiskRequestData > DiskRequestDataContainer;
/** Internal datatype used by the ChunkReaderWriter to keep track of requests
Arguments:
chunkID: id of the chunk we are dealing with
hMsg: the hopping message to send back to EE
token: the token to send back
*/
<?php
grokit\create_data_type("CRWRequest", "Data", ['chunkID' => 'off_t'], ['hMsg' => 'HoppingDataMsg', 'token' => 'GenericWorkToken']);
?>
typedef Keyify<off_t> KOff_t; // keyified off_t
typedef EfficientMap< KOff_t, CRWRequest > IDToRequestMap;
#endif // DISK_IO_DATA_
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
require_once 'DataFunctions.php';
?>
#ifndef _BULK_LOADER_INTERNAL_H_
#define _BULK_LOADER_INTERNAL_H_
#include "ContainerTypes.h"
/* Data types used by Bulk Loader to do its internal work. Should be
of no interest to the rest of the system */
<?php
grokit\create_data_type("TextLoaderDS", "Data", ['stream' => 'FILE*', 'file' => 'std::string'], []);
?>
// container for the above
typedef TwoWayList<TextLoaderDS> TextLoaderDSContainer;
#endif // _BULK_LOADER_INTERNAL_H_
<?php
grokit\create_data_type("GISTProduceResultsWorkFunc", "WorkFuncWrapper", [], [], true);
?>
<?php
grokit\create_data_type("GISTProduceStateWorkFunc", "WorkFuncWrapper", [], [], true);
?>
<?php
grokit\create_data_type("CacheChunkWorkFunc", "WorkFuncWrapper", [], [], true);
?>
<?php
grokit\create_data_type("GSEPreProcessWorkFunc", "WorkFuncWrapper", [], [], true);
?>
<?php
grokit\create_data_type("GSEProcessReadOnlyWorkFunc", "WorkFuncWrapper", [], [], true);
?>
<?php
grokit\create_data_type("ClusterProcessChunkWorkFunc", "WorkFuncWrapper", [], [], true);
?>
<?php
grokit\generate_deserializer('WorkFuncWrapper');
?>
#endif
// Produces results for this GIST.
<?php
grokit\create_data_type("GISTProduceResultsWD", "WorkDescription", ['fragmentNo' => 'int'], ['whichOne' => 'QueryExit', 'gist' => 'GLAState']);
?>
<?php
grokit\create_data_type("GIProduceChunkWD", "WorkDescription", [], ['gi' => 'GLAState', 'stream_info' => 'GIStreamProxy', 'queriesCovered' => 'QueryIDSet']);
?>
/***** GSE Work Descriptions *****/
// Pre-processing
<?php
grokit\create_data_type("GSEPreProcessWD", "WorkDescription", [], ['whichQueryExits' => 'QueryExitContainer', 'requiredStates' => 'QueryToGLASContMap']);
?>
// Processing a read-only request
<?php
grokit\create_data_type("GSEProcessWD", "WorkDescription", [], ['whichQueryExits' => 'QueryExitContainer', 'gseStates' => 'QueryToGLAStateMap', 'constStates' => 'QueryToGLAStateMap', 'request' => 'ServiceData']);
?>
/****** Cluster Work Descriptions ******/
// Process Chunk
<?php
grokit\create_data_type('ClusterProcessChunkWD', 'WorkDescription', [], ['chunk' => 'Chunk']);
?>
#endif // WORK_DESCRIPTION_H
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
require_once 'DataFunctions.php';
?>
#include <inttypes.h>
#include <string>
#include "Data.h"
#include "TwoWayList.h"
/** Container for counters
*/
<?php
grokit\create_data_type("PCounter", "DataC", ['name' => 'std::string', 'value' => 'int64_t', 'group' => 'std::string'], [], true);
?>
typedef TwoWayList <PCounter> PCounterList;
#endif // _PROF_MSG_DATA_H_
/** Text Loader
Arguments:
files: names of files that we are bulkloading
queries: the queries that are going to tag the produced chunks
*/
<?php
grokit\create_data_type("TextLoaderConfigureData", "WayPointConfigureData", ['files' => 'StringContainer'], ['queries' => 'QueryExitContainer'], true);
?>
<?php
grokit\create_data_type("TileJoinConfigureData", "WayPointConfigureData", ['lhsrelName' => 'std::string', 'rhsrelName' => 'std::string'], ['JoinWP' => 'WayPointID', 'queryColumnsMapLhs' => 'QueryExitToSlotsMap', 'queryColumnsMapRhs' => 'QueryExitToSlotsMap', 'columnsToSlotsPairLhs' => 'SlotPairContainer', 'columnsToSlotsPairRhs' => 'SlotPairContainer'], true);
?>
<?php
grokit\create_data_type("GIConfigureData", "WayPointConfigureData", ['files' => 'StringContainer'], ['queries' => 'QueryExitContainer'], true);
?>
<?php
grokit\generate_deserializer('WayPointConfigureData');
?>
// this is the list of way point configurations sent into the execution engine
typedef TwoWayList<WayPointConfigureData> WayPointConfigurationList;
#endif
// Copyright 2013 Tera Insights, LLC. All Rights Reserved.
require_once 'DataFunctions.php';
?>
#ifndef _EXEC_ENGINE_TASKS_H_
#define _EXEC_ENGINE_TASKS_H_
#include <cinttypes>
#include <string>
#include "TwoWayList.h"
// Base class for Execution Engine Tasks
<?php
grokit\create_base_data_type("Task", "DataC", [], [], true, true);
?>
typedef TwoWayList<Task> TaskList;
// Delete Relation
<?php
grokit\create_data_type("DeleteRelationTask", "Task", ['relation' => 'std::string'], [], true);
?>
<?php
grokit\generate_deserializer('Task');
?>
#endif // _EXEC_ENGINE_TASKS_H_
?>
// ProduceResults work function will produce a ChunkContainer
// Result of a GI reading data from a file.
<?php
grokit\create_data_type("GIProduceChunkRez", "ExecEngineData", [], ['stream' => 'GIStreamProxy', 'gi' => 'GLAState', 'chunk' => 'ChunkContainer']);
?>
/***** Return types for GSEs *****/
// Results from preprocessing
<?php
grokit\create_data_type("GSEPreProcessRez", "ExecEngineData", [], ['constStates' => 'QueryToGLAStateMap']);
?>
// Results from processing
<?php
grokit\create_data_type("GSEProcessRez", "ExecEngineData", [], ['gseStates' => 'QueryToGLAStateMap', 'result' => 'ServiceData']);
?>
/***** Return types for Cluster WP ******/
<?php
grokit\create_data_type("ClusterProcessChunkRez", "ExecEngineData", ['min' => 'int64_t', 'max' => 'int64_t'], []);
?>
#endif
typedef TwoWayList <GLAState> GLAStateContainer;
typedef EfficientMap <QueryID, GLAState> QueryToGLAStateMap;
typedef EfficientMap <QueryID, GLAStateContainer> QueryToGLASContMap;
typedef EfficientMap< QueryID, Swapify<int> > QueryIDToInt;
typedef EfficientMap< QueryID, Swapify<bool> > QueryIDToBool;
typedef TwoWayList<WayPointID> ReqStateList;
typedef EfficientMap<QueryID, ReqStateList> QueryToReqStates;
/** GLAPointer stores a pointer to a GLA state. This is an in-memory
pointer.
*/
<?php
grokit\create_data_type("GLAPtr", "GLAState", ['glaPtr' => 'void*'], [], false);
?>
/******* Data containing QueryIDSet and Chunk **/
<?php
grokit\create_base_data_type("CacheData", "DataC", ['queryIDs' => 'QueryIDSet'], ['cacheChunk' => 'Chunk'], false);
?>
typedef EfficientMap <ChunkID, CacheData> ChunkToCacheMap;
/** Data Structures for GIST Waypoints */
<?php
grokit\create_base_data_type("GISTWorkUnit", "DataC", [], ['gist' => 'GLAState', 'localScheduler' => 'GLAState', 'gla' => 'GLAState'], false);
// needs to go, lineage is a list of History objects (where each waypoint that
// obtains and then forwards on the message can add new history objects to the
// list), and data is the actual data in the message.
<?php
grokit\create_base_data_type("HoppingDataMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExitContainer', 'lineage' => 'HistoryList', 'data' => 'ExecEngineData']);
?>
// this macro defines direct messages, that are sent to a particular waypoint
<?php
grokit\create_base_data_type("DirectMsg", "Data", ['receiver' => 'WayPointID'], ['message' => 'Notification']);
?>
// this macro defines hopping downstream messages... these are a lot like hopping
// data messages, except that they contain Notification objects instead of data
<?php
grokit\create_data_type("HoppingDownstreamMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExitContainer', 'msg' => 'Notification']);
?>
// this macro defines hopping upstream messages... unlike the downstream messages,
// these have only one destination---the waypoint(s) that produce(s) data associated
// with a particular query exit
<?php
grokit\create_data_type("HoppingUpstreamMsg", "DataC", ['currentPos' => 'WayPointID'], ['dest' => 'QueryExit', 'msg' => 'Notification']);
?>
#endif
// set of sampled entries from all of the too-full segments
<?php
grokit\create_data_type("TooFullMessage", "Notification", [], ['whatWeFound' => 'HashSegmentSample']);
?>
// this is sent from the hash cleaner to a particular waypoint to notify the waypoint
// that it has now become disk-based
<?php
grokit\create_data_type("IKillYouMessage", "Notification", [], []);
?>
// when a join waypoint finlly dies (that is, when a "wounded" waypoint that is being emptied
// out of the hash table has its last bytes ever removed from the hash table) it sends this
// message to both the hash cleaner and to the writer letting them know that it has finally died
<?php
grokit\create_data_type("WayPointDeadMsg", "Notification", [], []);
?>
// messge sent from cleaner to a join with the hash
<?php
grokit\create_data_type("CentralHashMessage", "Notification", [], ['centralHash' => 'HashTable']);
?>
<?php
grokit\create_data_type("GetHashTable", "Notification", [], []);
?>
#endif
// this file contains all of the various work token types
#include "Data.h"
#include "DistributedQueue.cc"
// this macro defines the generic token type, which is sent to waypoints when they
// request the ability to do some work... tokens must also be sent back to the
// execution engine when the work is done. grantedTo is the identifier for the
// waypoint who was actually granted the right to use the token
<?php
grokit\create_base_data_type("GenericWorkToken", "Data", ['label' => 'int'], []);
?>
// this macro defines the CPU token type
<?php
grokit\create_data_type("CPUWorkToken", "GenericWorkToken", [], []);
?>
// this macro defines the Disk token type
<?php
grokit\create_data_type("DiskWorkToken", "GenericWorkToken", [], []);
?>
// this is used to send a list of disk tokens for writer waypoints to use
typedef DistributedQueue <DiskWorkToken> DiskWorkTokenQueue;
#endif
<?php
grokit\create_data_type("TextLoaderHistory", "History", ['whichChunk' => 'ChunkID', 'file' => 'std::string'], []);
?>
<?php
grokit\create_data_type("GIHistory", "History", ['whichChunk' => 'ChunkID', 'file' => 'std::string'], []);
?>
<?php
grokit\create_data_type("TileJoinScanHistory", "History", ['bucketID' => 'int', 'whichChunk' => 'ChunkID', 'isLHS' => 'bool', 'file' => 'std::string'], ['whichExits' => 'QueryExitContainer']);
?>
<?php
grokit\create_data_type("TileJoinWriteHistory", "History", [], []);
?>
<?php
grokit\create_data_type("TileJoinMergeHistory", "History", ['bucketID' => '__uint64_t'], ['chunkIDLHSList' => 'ChunkIDContainer', 'chunkIDRHSList' => 'ChunkIDContainer', 'whichExits' => 'QueryExitContainer']);
?>
<?php
grokit\create_data_type("CacheHistory", "History", ['whichChunk' => 'uint64_t'], []);
?>
#endif
