public function run($arg)
{
if (!$arg) {
return;
}
$arg = (object) $arg;
$SQ = new ShardQuery();
if (!isset($arg->use_fifo)) {
$arg->use_fifo = true;
}
$SL = new ShardLoader($SQ, $arg->loadspec['delimiter'], $arg->loadspec['enclosure'], $arg->loadspec['line_terminator'], $arg->use_fifo);
$result = $SL->load_segment($arg->path, $arg->table, $arg->start, $arg->end, $arg->loadspec['columns_str'], $arg->loadspec['set_str'], $arg->loadspec['ignore'], $arg->loadspec['replace']);
if (is_array($result)) {
register_completion($arg->job_id, 'loader', 'error', print_r($result, true));
return false;
}
register_completion($arg->job_id, 'loader', 'ok', 'no errors');
return true;
}
protected function process_load_data($sql)
{
$file_name = false;
$table_name = "";
$fields_terminated_by = ",";
$lines_terminated_by = "\\n";
$fields_escaped_by = "\\";
$fields_optionally_enclosed = false;
$fields_enclosed_by = '';
$local = false;
$charset = "latin1";
$ignore = false;
$replace = false;
$columns_str = null;
$set_str = null;
$lines_starting_by = "";
$chunk_size = 16 * 1024 * 1024;
$sql = trim($sql, ';');
$regex = "/[A-Za-z_.]+\\(.*?\\)+|\\(.*?\\)+|\"(?:[^\"]|\"|\"\")*\"+|'[^'](?:|\\'|'')*'+|`(?:[^`]|``)*`+|[^ ,]+ |,/x";
$regex = trim($regex);
preg_match_all($regex, $sql, $matches, PREG_OFFSET_CAPTURE);
$tokens = $matches[0];
if (strtolower($tokens[0][0]) != 'load') {
$this->errors[] = 'Malformed LOAD DATA statement. LOAD is missing.';
return false;
}
$skip_next = false;
$line_options = false;
$field_options = false;
$ignore_lines = 0;
$set_pos_at = 0;
$past_infile = false;
foreach ($tokens as $key => $token) {
if ($token[0] == "'") {
continue;
}
if ($token[0] == "(") {
$columns_str = $token;
continue;
}
if ($skip_next) {
$skip_next = false;
continue;
}
$token = $token[0];
switch (strtolower($token)) {
case 'load':
case 'data':
case 'low_priority':
case 'concurrent':
break;
/* allow configuration of the size of each chunk from the file */
/* allow configuration of the size of each chunk from the file */
case 'chunksize':
$chunk_size = $tokens[$key + 1][0];
$base = substr($chunk_size, 0, strlen($chunk_size) - 1);
$lastchar = substr($chunk_size, -1);
switch (strtolower($lastchar)) {
case 'b':
$chunk_size = $base;
break;
case 'k':
$chunk_size = $base * 1024;
break;
case 'm':
$chunk_size = $base * 1024 * 1024;
break;
case 'g':
$chunk_size = $base * 1024 * 1024 * 1024;
break;
}
$skip_next = true;
break;
case 'replace':
$replace = true;
break;
case 'ignore':
if ($past_infile) {
$ignore_lines = $tokens[$key + 2][0];
} else {
$ignore = true;
}
break;
case 'character':
$charset = $tokens[$key + 2][0];
$skip_next = true;
break;
case 'local':
$local = true;
break;
case 'infile':
$past_infile = true;
$file_name = $this->clean_token($tokens[$key + 1][0]);
$skip_next = true;
break;
case 'into':
break;
case 'table':
$table_name = $tokens[$key + 1][0];
$skip_next = true;
break;
case 'fields':
case 'columns':
$field_options = true;
break;
case 'terminated':
if ($line_options) {
$lines_terminated_by = $this->clean_token($tokens[$key + 2][0]);
} else {
$fields_terminated_by = $this->clean_token($tokens[$key + 2][0]);
}
break;
case 'by':
$skip_next = true;
break;
case 'optionally':
$fields_optionally_enclosed = true;
break;
case 'enclosed':
$fields_enclosed_by = $this->clean_token($tokens[$key + 2][0]);
break;
case 'lines':
$line_options = true;
$field_options = false;
break;
case 'starting':
if ($line_options) {
$lines_starting_by = $tokens[$key + 2][0];
}
break;
case 'set':
$set_pos_at = $tokens[$key][1];
break 2;
#leave for loop
}
}
if ($table_name === "") {
$this->errors[] = "LOAD DATA requires a table name";
return false;
}
if ($file_name === "") {
$this->errors[] = "LOAD DATA requires a file name";
return false;
}
if ($replace) {
$replace = "REPLACE";
} else {
$replace = "";
}
if ($ignore) {
$ignore = "IGNORE";
} else {
$ignore = "";
}
if ($set_pos_at) {
$set_str = substr($sql, $set_pos_at);
} else {
$set_str = "";
}
$lines_terminated_by = str_replace("\\n", "\n", $lines_terminated_by);
#TODO: Handle setting the shard_key in the SET clause
$SL = new ShardLoader($this, $fields_terminated_by, $fields_enclosed_by, $lines_terminated_by, true, $chunk_size, $charset, $ignore, $replace, $lines_starting_by, $fields_escaped_by);
if ($local) {
if (!$SL->load($file_name, $table_name, null, null, $columns_str, $set_str, $ignore, $replace)) {
$this->errors = $SL->errors;
return false;
}
} else {
if (!(strstr($file_name, 's3://') || strstr($file_name, 'http://') || strstr($file_name, 'https://'))) {
$file_name = "{$this->state->shared_path}/{$file_name}";
}
if (!$SL->load_gearman($file_name, $table_name, null, null, $columns_str, $set_str, $ignore, $replace)) {
$this->errors = $SL->errors;
}
}
unset($SL);
return false;
}