function PatternMatcherOnig($t_args, $inputs)
{
grokit_assert(\count($inputs) == 1, 'PatternMatcherOnig GF only supports 1 input!');
$pattern = get_first_key($t_args, ['pattern']);
$inName = array_keys($inputs)[0];
$inType = array_get_index($inputs, 0);
$inTypeString = $inType->name();
$validTypes = ['BASE::STRING_LITERAL'];
grokit_assert(in_array($inTypeString, $validTypes), 'Unsupported input type ' . $inTypeString);
$className = generate_name('PatternMatcherOnigGF');
?>
class <?php
echo $className;
?>
{
PatternMatcherOnig matcher;
public:
<?php
echo $className;
?>
() :
matcher("<?php
echo $pattern;
?>
")
{ }
bool Filter( const <?php
echo $inType;
?>
& <?php
echo $inName;
?>
) {
return matcher.Match(<?php
echo $inName;
?>
);
}
};
<?php
return ['kind' => 'GF', 'name' => $className, 'input' => $inputs, 'user_headers' => ['PatternMatcherOnig.h']];
}
* | This MySource Matrix CMS file is Copyright (c) Squiz Pty Ltd |
* | ABN 77 084 670 600 |
* +--------------------------------------------------------------------+
* | IMPORTANT: Your use of this Software is subject to the terms of |
* | the Licence provided in the file licence.txt. If you cannot find |
* | this file please contact Squiz (www.squiz.com.au) so we may provide|
* | you a copy. |
* +--------------------------------------------------------------------+
*
* $Id: session.php,v 1.9 2012/08/30 01:09:21 ewang Exp $
*
*/
// include init.in, which will call $GLOBALS['SQ_SYSTEM']->init() and start a session
require_once '../../include/init.inc';
header("content-type: application/javascript");
$site_network_id = array_get_index($_GET, 'site_network', '0');
$site_network = null;
$primary_url = '';
$session_handler = $GLOBALS['SQ_SYSTEM']->getSessionHandlerClassName();
if ($site_network_id) {
$site_network =& $GLOBALS['SQ_SYSTEM']->am->getAsset($site_network_id);
if (!is_null($site_network)) {
$primary_url = $site_network->getPrimaryURL();
}
}
if ($primary_url == sq_web_path('root_url')) {
// if the actual script execution is happening in the primary
// url, we want to update the time to persuade the overriding
// of the secondary session files
if (isset($_GET['in_primary']) && $_GET['in_primary']) {
$_SESSION['SQ_SESSION_TIMESTAMP'] = time();
//end if
// Check for a lock file
if (file_exists(SQ_TEMP_PATH . '/funnelback.indexer')) {
trigger_localised_error('FNB0019', translate('Lockfile found. Scripts cannot be run concurrently.'), E_USER_WARNING);
exit;
}
//end if
// Create a lock file
touch(SQ_TEMP_PATH . '/funnelback.indexer');
// Start indexing
$hh = $GLOBALS['SQ_SYSTEM']->getHipoHerder();
$vars = array('collections' => empty($root_collection) ? array() : $root_collection);
$errors = $hh->freestyleHipo('hipo_job_funnelback_reindex', $vars, SQ_PACKAGES_PATH . '/funnelback/hipo_jobs');
if (!empty($errors)) {
echo 'Funnelback Reindexing FAILED' . "\n";
foreach ($errors as $error) {
$line = array_get_index($error, 'message', '');
if (!empty($line)) {
echo $line . "\n";
}
//end if
}
//end foreach
}
//end if
// Remove if finished AND failed
if (file_exists(SQ_TEMP_PATH . '/funnelback.indexer')) {
unlink(SQ_TEMP_PATH . '/funnelback.indexer');
}
//end if
$GLOBALS['SQ_SYSTEM']->restoreRunLevel();
if (php_sapi_name() != 'cli') {
trigger_error('This script can only be run from the command line', E_USER_ERROR);
exit;
}
//end if
$SYSTEM_ROOT = isset($_SERVER['argv'][1]) ? $_SERVER['argv'][1] : '';
if (empty($SYSTEM_ROOT) || !is_readable($SYSTEM_ROOT)) {
trigger_error('You need to specify a path to Matrix as the first parameter', E_USER_ERROR);
exit;
}
//end if
define('SQ_SYSTEM_ROOT', $SYSTEM_ROOT);
require_once SQ_SYSTEM_ROOT . '/core/include/init.inc';
// Check the script is being run from same user as apache
$apache_user = fileowner(SQ_SYSTEM_ROOT . '/data/private/conf/main.inc');
$current_user = array_get_index($_ENV, 'USER', '');
if ($apache_user != $current_user) {
// trigger_error('This script needs to run as the apache user', E_USER_ERROR);
// exit();
}
//end if
$funnelback_manager = $GLOBALS['SQ_SYSTEM']->am->getSystemAsset('funnelback_manager');
// Check the base install for paths and permissions
$valid = $funnelback_manager->checkInstalled();
if ($valid) {
echo "Looks ok from here\n";
} else {
echo "Funnelback has not been installed correctly\n";
}
//end if
exit;
/**
* A GLA that estimates the cardinality of a dataset using the HyperLogLog
* algorithm, with a configurable number of bins.
*/
function HyperLogLog(array $t_args, array $input, array $output)
{
$debug = get_default($t_args, 'debug', 0);
grokit_assert(\count($output) == 1, 'HyperLogLog produces only 1 value, ' . \count($output) . ' outputs given.');
$outputName = array_keys($output)[0];
$outputType = array_get_index($output, 0);
if (is_null($outputType)) {
$outputType = lookupType('BASE::BIGINT');
}
$output[$outputName] = $outputType;
grokit_assert($outputType->is('numeric'), 'BloomFilter output must be numeric!');
$exp = get_first_key_default($t_args, ['bins.exponent'], 4);
grokit_assert(is_integer($exp), 'HyperLogLog bins.exponent must be an integer');
// Set limit of 2^24 bins, because states past 16MB start to get silly
grokit_assert($exp >= 4 && $exp < 24, 'HyperLogLog bins.exponent must be in range [4, 24]');
$useBuiltinCtz = get_default($t_args, 'use.builtin.ctz', true);
$ctzFunc = $useBuiltinCtz ? '__builtin_ctzl' : 'ctz';
$bins = pow(2, $exp);
// Determine the value of alpha based on $exp
switch ($exp) {
case 4:
$alpha = 0.673;
break;
case 5:
$alpha = 0.697;
break;
case 6:
$alpha = 0.709;
break;
default:
$alpha = 0.7213000000000001 / (1 + 1.079 / $bins);
}
$className = generate_name('HyperLogLog');
?>
class <?php
echo $className;
?>
{
// Number of bins for registers
static constexpr const size_t NUM_BINS = <?php
echo $bins;
?>
;
// Number of bits used to index into registers, log2(NUM_BINS)
static constexpr const size_t INDEX_BITS = <?php
echo $exp;
?>
;
// Mask used to obtain register index from hash value
static constexpr const size_t INDEX_MASK = NUM_BINS - 1;
// Alpha coefficient used to correct cardinality estimate. Based on NUM_BINS.
static constexpr const long double ALPHA = <?php
echo $alpha;
?>
;
// Value of cardinality estimate after which we must apply the
// large range correction
static constexpr const long double LARGE_BREAKPOINT = (1.0 / 30.0) * <?php
echo pow(2, 32);
?>
;
// Constants for population count
static constexpr const uint64_t m1 = 0x5555555555555555;
static constexpr const uint64_t m2 = 0x3333333333333333;
static constexpr const uint64_t m4 = 0x0f0f0f0f0f0f0f0f;
static constexpr const uint64_t h01 = 0x0101010101010101;
// The registers
std::array<unsigned char, NUM_BINS> registers;
// A count used to remember how many tuples were processed, mostly for debugging.
size_t count;
public:
<?php
echo $className;
?>
(void) : registers() {
for( auto & elem : registers ) {
elem = 0;
}
}
~<?php
echo $className;
?>
() { }
int popcount(uint64_t x) {
// Put count of each 2 bits into those 2 bits
x -= (x >> 1) & m1;
// Put count of each 4 bits into those 4 bits
x = (x & m2) + ((x >> 2) & m2);
// Put count of each 8 bits into those 8 bits
x = (x + (x >> 4)) & m4;
// Returns left 8 bits of x + (x << 8) + (x << 16) + ...
return (x * h01) >> 56;
}
int ctz(int64_t x) {
return popcount((x & -x) - 1);
}
void AddItem( <?php
echo const_typed_ref_args($input);
?>
) {
count++;
uint64_t hashVal = H_b;
<?php
foreach ($input as $name => $type) {
?>
hashVal = CongruentHash(Hash(<?php
echo $name;
?>
), hashVal);
<?php
}
// for each input
?>
const size_t registerIndex = hashVal & INDEX_MASK;
uint64_t value = hashVal >> INDEX_BITS;
unsigned char nZeros = <?php
echo $ctzFunc;
?>
(value);
unsigned char & registerValue = registers[registerIndex];
registerValue = registerValue > nZeros ? registerValue : nZeros;
}
void AddState( <?php
echo $className;
?>
& other ) {
for( size_t i = 0; NUM_BINS > i; i++ ) {
unsigned char & rVal = registers[i];
unsigned char & oVal = other.registers[i];
rVal = rVal > oVal ? rVal : oVal;
}
}
void GetResult( <?php
echo $outputType;
?>
& <?php
echo $outputName;
?>
) {
// Compute harmonic sum of registers and correct by alpha
long double cardEst = 0;
size_t nZeroRegisters = 0;
for( auto elem : registers ) {
long double power = - static_cast<long double>(elem);
cardEst += std::pow(2.0, power);
if( elem == 0 )
nZeroRegisters++;
}
const long double nBins = static_cast<long double>(NUM_BINS);
const long double zeroBins = static_cast<long double>(nZeroRegisters);
cardEst = 1 / cardEst;
cardEst *= ALPHA * nBins * nBins;
long double cardinality = cardEst;
if( (cardEst < 2.5 * NUM_BINS) ) { //>
// Possible small range correction
if( nZeroRegisters > 0 ) {
// Small range correction
cardinality = nBins * std::log(nBins / zeroBins);
}
}
// TODO: Figure out if the large range correction is needed for 64-bit
// hashes.
<?php
echo $outputName;
?>
= cardinality;
}
};
<?php
$system_headers = ['cmath', 'array', 'cinttypes'];
if ($debug > 0) {
$system_headers[] = 'iostream';
}
return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'user_headers' => ['HashFunctions.h'], 'system_headers' => $system_headers];
}
public function __construct($kind, $hash, $name, $value, array $args, array $targs)
{
parent::__construct($kind, $hash, $name, $value, $args, $targs);
if (array_key_exists('required_states', $args)) {
$this->req_states = $args['required_states'];
}
if (array_key_exists('generated_state', $args)) {
$this->gen_state = $args['generated_state'];
}
$this->configurable = get_default($args, 'configurable', false);
if (array_key_exists('generated_states', $args)) {
grokit_warning('"generated_states" type information parameter is deprecated. Please use "generated_state" instead.');
$this->gen_states = ensure_unique_names($args['generated_states'], 'gen_state');
if (is_null($this->gen_state) && \count($this->gen_states) == 1) {
$this->gen_state = array_get_index($this->gen_states, 0);
}
}
if (array_key_exists('constructor_args', $args)) {
grokit_warning('"constructor_args" type information parameter is deprecated.');
$this->constructor_args = ensure_unique_names($args['constructor_args'], 'const_arg');
}
}
*
* $Id: keywords.php,v 1.1 2013/09/24 01:10:35 ewang Exp $
*
*/
require_once dirname(__FILE__) . '/../../../../core/include/init.inc';
require_once dirname(__FILE__) . '/../../../../core/lib/html_form/html_form.inc';
$assetid = array_get_index($_GET, 'assetid');
if (is_null($assetid)) {
return FALSE;
}
assert_valid_assetid($assetid);
$asset = $GLOBALS['SQ_SYSTEM']->am->getAsset($assetid);
if (is_null($asset) || !$asset->writeAccess()) {
exit;
}
$all = array_get_index($_REQUEST, 'all', FALSE);
require_once dirname(__FILE__) . '/../../../../core/include/backend_outputter.inc';
$o = new Backend_Outputter();
$o->openSection('Metadata Keyword Replacements');
$o->openField('', 'wide_col');
?>
<p>The following keyword replacements may be used for metadata values. Note that the percentage signs (%) are required.</p>
<?php
$o->openSection('Asset Information');
$o->openField('', 'wide_col');
?>
<table class="sq-backend-table compact">
<tr><th style="width: 250px;">Keyword</th><th>Description</th></tr>
$PERMISSION = isset($_SERVER['argv'][4]) ? $_SERVER['argv'][4] : '';
if (empty($PERMISSION) || !array_key_exists($PERMISSION, $available_permissions)) {
echo "ERROR: You need to supply a permission as the fourth argument\n";
exit;
}
$GRANTED = isset($_SERVER['argv'][5]) && $_SERVER['argv'][5] == 'y' ? '1' : '0';
$CHILDREN = isset($_SERVER['argv'][6]) && $_SERVER['argv'][6] == 'y' ? TRUE : FALSE;
// Start the process
$root_user =& $GLOBALS['SQ_SYSTEM']->am->getSystemAsset('root_user');
$GLOBALS['SQ_SYSTEM']->setCurrentUser($root_user);
$GLOBALS['SQ_SYSTEM']->setRunLevel(SQ_RUN_LEVEL_FORCED);
echo 'START PERMISSION CHANGE' . "\n";
$hh = $GLOBALS['SQ_SYSTEM']->getHipoHerder();
$vars = array('permission_changes' => array(array('permission' => $available_permissions[$PERMISSION], 'granted' => $GRANTED, 'assetids' => array($ROOT_NODE), 'userid' => $USER, 'previous_access' => NULL, 'cascades' => $CHILDREN)));
$errors = $hh->freestyleHipo('hipo_job_edit_permissions', $vars);
echo 'FINISHED';
if (!empty($errors)) {
echo "... But with errors!\n";
foreach ($errors as $error) {
echo "\t" . ($line = array_get_index($error, 'message', ''));
if (!empty($line)) {
echo "\t" . $line . "\n";
}
//end if
}
//end foreach
} else {
echo "\n";
}
//end if
$GLOBALS['SQ_SYSTEM']->restoreRunLevel();
*
*/
/**
* Search and replaces given text in the WYSIWYG
*
* @author Chiranjivi Upreti <*****@*****.**>
* @version $Revision: 1.5 $
* @package MySource_Matrix
*/
require_once dirname(__FILE__) . '/../../../../core/include/init.inc';
require_once SQ_LIB_PATH . '/html_form/html_form.inc';
require_once SQ_FUDGE_PATH . '/var_serialise/var_serialise.inc';
if (empty($GLOBALS['SQ_SYSTEM']->user) || !($GLOBALS['SQ_SYSTEM']->user->canAccessBackend() || $GLOBALS['SQ_SYSTEM']->user->type() == 'simple_edit_user' || method_exists($GLOBALS['SQ_SYSTEM']->user, 'isShadowSimpleEditUser') && $GLOBALS['SQ_SYSTEM']->user->isShadowSimpleEditUser())) {
exit;
}
$_GET['name'] = array_get_index($_GET, 'name', '');
?>
<!doctype html>
<html style="height: 280px;">
<head>
<title>Search and Replace</title>
<link rel="stylesheet" type="text/css" href="<?php
echo sq_web_path('lib') . '/web/css/edit.css';
?>
" />
<link rel="stylesheet" type="text/css" href="<?php
echo sq_web_path('root_url');
?>
/__fudge/wysiwyg/core/popup.css" />
<script type="text/javascript" src="../../core/popup.js"></script>
if ($count % 100 == 0) {
echo '.';
}
// Key fields data
$key_values = array();
foreach ($key_fields as $key_field) {
$temp_key_v = array_get_index($record, $key_field, NULL);
if (is_null($temp_key_v)) {
// Primary key field must be there
continue 2;
}
$key_values[$key_field] = $temp_key_v;
}
//end foreach
// Value field to check
$org_value = array_get_index($record, $value_field, '');
if (empty($org_value)) {
continue;
}
// Replace the "./?a=xxx" links in the content
$new_value = _rewrite_assetid_links($org_value, $record, $table == 'ast_mdata_val');
if ($new_value !== FALSE && !$report_mode) {
// Generate update sql
$bind_vars = array();
$set_sql = $value_field . '=:' . $value_field . '_v';
$bind_vars[$value_field . '_v'] = $new_value;
$where_sql = array();
foreach ($key_values as $field_name => $value) {
$where_sql[] = $field_name . '=:' . $field_name . '_k';
$bind_vars[$field_name . '_k'] = $value;
}
}
// Cut it down to a certain number of characters anyway
$value = ellipsisize($value, 120);
$value = preg_replace('/(' . addslashes(implode('|', $words)) . ')/i', '<span class="sq-backend-search-results-highlight">$1</span>', $value);
// remove \r and replace \n with line breaks
$this_detail[] = $name_detail . '<br/><em>' . str_replace("\r", '', str_replace("\n", '<br/>', $value)) . '</em>';
}
//end foreach
}
//end foreach
$asset_name = $GLOBALS['SQ_SYSTEM']->am->getAssetInfo(array($result_assetid), 'image', FALSE, 'name');
$asset_title = $GLOBALS['SQ_SYSTEM']->am->getAttributeValuesByName('title', 'image', array($result_assetid));
$asset_alt = $GLOBALS['SQ_SYSTEM']->am->getAttributeValuesByName('alt', 'image', array($result_assetid));
$asset_width = $GLOBALS['SQ_SYSTEM']->am->getAttributeValuesByName('width', 'image', array($result_assetid));
$asset_height = $GLOBALS['SQ_SYSTEM']->am->getAttributeValuesByName('height', 'image', array($result_assetid));
$result_list[] = array('tag_line' => get_asset_tag_line($result_assetid, 'javascript:set_asset_finder_from_search(\'' . $result_assetid . '\', \'' . htmlspecialchars(array_get_index($asset_title, $result_assetid, ''), ENT_QUOTES) . '\', \'\', \'0\', \'' . array_get_index($asset_name, $result_assetid, '') . '\', \'' . addslashes(array_get_index($asset_alt, $result_assetid, '')) . '\', \'' . array_get_index($asset_width, $result_assetid, '') . '\', \'' . array_get_index($asset_height, $result_assetid, '') . '\');'), 'detail' => implode($this_detail, '<br/>'));
}
//end foreach
}
//end if
}
//end else
// Are there any results? If not, put in a "search failed" box, otherwise
// build the results box
if (empty($results) && empty($found_asset_line)) {
$box_title = translate('Search Failed');
$html = sprintf(translate('Search for "%s" found nothing. Try Again'), addslashes($search_for));
$style_base = 'search-failed';
} else {
$box_title = translate('Search Results');
if (!empty($found_asset_line)) {
echo 'Resuming HIPO Job ';
echo ucwords(str_replace('_', ' ', $source_job_type)) . "\t";
// Do itttttt
$status = $source_job->process();
if ($status) {
echo '[ OK ]';
} else {
echo '[ !! ]';
}
//end if
echo "\n";
break;
case 'k':
// Kill kill kill
$source_code_name = array_get_index($source_jobs[$actual_choice], 'source_code_name', '');
$source_job_type = array_get_index($source_jobs[$actual_choice], 'job_type', '');
if (empty($source_code_name)) {
exit;
}
$source_job = $hh->getJob($source_code_name);
if (is_null($source_job)) {
exit;
}
echo 'Aborting HIPO Job ';
echo ucwords(str_replace('_', ' ', $source_job_type)) . "\n";
$source_job->abort();
break;
case 'q':
default:
exit;
}
* @author Benjamin Pearson <*****@*****.**>
* @version $Revision: 1.4 $
* @package MySource_Matrix
*/
require_once dirname(__FILE__) . '/../../../../core/include/init.inc';
require_once SQ_LIB_PATH . '/html_form/html_form.inc';
require_once SQ_LIB_PATH . '/backend_search/backend_search.inc';
if (empty($GLOBALS['SQ_SYSTEM']->user) || !($GLOBALS['SQ_SYSTEM']->user->canAccessBackend() || $GLOBALS['SQ_SYSTEM']->user->type() == 'simple_edit_user' || method_exists($GLOBALS['SQ_SYSTEM']->user, 'isShadowSimpleEditUser') && $GLOBALS['SQ_SYSTEM']->user->isShadowSimpleEditUser())) {
exit;
}
if (Backend_Search::isAvailable()) {
$quick_search_for_text = translate('Enter keywords, asset ID or URL');
} else {
$quick_search_for_text = translate('Enter asset ID or URL');
}
$search_for = trim(array_get_index($_GET, 'quick-search-for', ''));
// If we are searching for something
if ($search_for != '') {
// check for a url first
$asset_by_url = $GLOBALS['SQ_SYSTEM']->am->getAssetFromURL('', strip_url($search_for, TRUE), TRUE, TRUE);
if (!empty($asset_by_url) && ($asset_by_url->type() != 'file' || !$asset_by_url->readAccess())) {
$asset_by_url = NULL;
}
if (assert_valid_assetid($search_for, '', TRUE, FALSE)) {
$asset_by_id = $GLOBALS['SQ_SYSTEM']->am->getAsset($search_for, '', TRUE);
if (!empty($asset_by_id) && ($asset_by_id->type() != 'file' || !$asset_by_id->readAccess())) {
$asset_by_id = NULL;
}
}
// Only search for files (strict type check)
$results = Backend_Search::processSearch($search_for, array(), array('file' => FALSE));
/**
* A GLA that counts the number of distinct elements by keeping track of the
* distinct elements.
*
* Unless an exact count of the distinct is absolutely needed, consider using
* an approximation of the distinct, such as a Bloom Filter.
*/
function CountDistinct(array $t_args, array $input, array $output)
{
grokit_assert(\count($output) == 1, 'CountDistinct should have only 1 output, ' . \count($output) . 'given');
$outputName = array_keys($output)[0];
$outputType = array_get_index($output, 0);
if (is_null($outputType)) {
$outputType = lookupType('BASE::BIGINT');
}
$output[$outputName] = $outputType;
grokit_assert($outputType->is('numeric'), 'CountDistinct output must be numeric!');
$useMCT = get_default($t_args, 'use.mct', true);
$keepHashes = get_default($t_args, 'mct.keep.hashes', false);
$initSize = get_default($t_args, 'init.size', 65536);
$nullCheck = get_default($t_args, 'null.check', false);
grokit_assert(is_bool($useMCT), 'CountDistinct use.mct argument must be boolean');
grokit_assert(is_integer($initSize), 'Distinct init.size argument must be an integer');
grokit_assert($initSize > 0, 'Distinct init.size argument must be positive');
grokit_assert(is_bool($keepHashes), 'CountDistinct mct.keep.hashes argument must be boolean');
$distTmpArgs = ['use.mct' => $useMCT, 'init.size' => $initSize, 'mct.keep.hashes' => $keepHashes, 'null.check' => $nullCheck];
$gla = lookupGLA('BASE::DISTINCT', $distTmpArgs, $input, $input);
$className = generate_name('CountDistinct');
?>
class <?php
echo $className;
?>
{
using Distinct = <?php
echo $gla->value();
?>
;
Distinct distinctGLA;
public:
<?php
echo $className;
?>
(void):
distinctGLA()
{ }
~<?php
echo $className;
?>
(void) { }
void AddItem(<?php
echo const_typed_ref_args($input);
?>
) {
distinctGLA.AddItem(<?php
echo args($input);
?>
);
}
void AddState(<?php
echo $className;
?>
& o) {
distinctGLA.AddState(o.distinctGLA);
}
void GetResult(<?php
echo $outputType;
?>
& <?php
echo $outputName;
?>
) {
<?php
echo $outputName;
?>
= distinctGLA.get_countDistinct();
}
};
<?php
return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single'];
}
MatrixDAL::executeSql('ALTER TABLE sq_suite_product DROP COLUMN knows_me_as');
// Populate data into not null columns.
MatrixDAL::executeSql("UPDATE sq_suite_product SET suiteid=sq_suite_seq.nextVal");
MatrixDAL::executeSql("UPDATE sq_suite_product SET url=''");
$products = MatrixDAL::executeSqlAssoc('SELECT suiteid, connection FROM sq_suite_product');
foreach ($products as $product) {
$suiteid = array_get_index($product, 'suiteid', NULL);
$connection = array_get_index($product, 'connection', NULL);
if ($suiteid === NULL || $connection === NULL) {
continue;
}
$connection = @unserialize($connection);
if ($connection === FALSE) {
continue;
}
$url = array_get_index($connection, 'url', NULL);
if ($url === NULL) {
continue;
}
unset($connection['url']);
$query = MatrixDAL::preparePdoQuery('UPDATE sq_suite_product SET url=:url, connection=:connection WHERE suiteid=:id');
MatrixDAL::bindValueToPdo($query, 'url', $url);
MatrixDAL::bindValueToPdo($query, 'connection', serialize($connection));
MatrixDAL::bindValueToPdo($query, 'id', $suiteid);
MatrixDAL::execPdoQuery($query);
}
//end foreach
// Set the not null constraint on columns.
MatrixDAL::executeSql('ALTER TABLE sq_suite_product MODIFY suiteid NOT NULL');
MatrixDAL::executeSql('ALTER TABLE sq_suite_product MODIFY url NOT NULL');
// Set the new constraints and keys
function Max(array $t_args, array $input, array $output)
{
grokit_assert(\count($output) >= 1, 'Max GLA produces at least one output!');
grokit_assert(\count($output) == \count($input), 'Max GLA should have the same number of inputs and outputs');
$nValues = \count($output);
$inputNames = array_keys($input);
$outputNames = array_keys($output);
// Outputs should be the same type as the inputs
for ($index = 0; $index < $nValues; $index++) {
array_set_index($output, $index, array_get_index($input, $index));
}
$name = generate_name('Max_');
?>
class <?php
echo $name;
?>
{
uintmax_t count;
<?php
foreach ($output as $k => $v) {
?>
<?php
echo $v;
?>
_<?php
echo $k;
?>
;
<?php
}
// foreach output
?>
public:
<?php
echo $name;
?>
() :
<?php
foreach ($output as $k => $v) {
?>
_<?php
echo $k;
?>
(),
<?php
}
// foreach output
?>
count(0)
{ }
void AddItem( <?php
echo const_typed_ref_args($input);
?>
) {
if( count > 0 ) {
<?php
for ($index = 0; $index < $nValues; $index++) {
?>
_<?php
echo $outputNames[$index];
?>
= std::max(_<?php
echo $outputNames[$index];
?>
, <?php
echo $inputNames[$index];
?>
);
<?php
}
// foreach value
?>
} else {
<?php
for ($index = 0; $index < $nValues; $index++) {
?>
_<?php
echo $outputNames[$index];
?>
= <?php
echo $inputNames[$index];
?>
;
<?php
}
// foreach value
?>
}
count++;
}
void AddState( <?php
echo $name;
?>
& o ) {
if (count > 0 && o.count > 0) {
<?php
for ($index = 0; $index < $nValues; $index++) {
?>
_<?php
echo $outputNames[$index];
?>
= std::max(_<?php
echo $outputNames[$index];
?>
, o._<?php
echo $outputNames[$index];
?>
);
<?php
}
// foreach value
?>
} else if(o.count > 0) { // count == 0
<?php
for ($index = 0; $index < $nValues; $index++) {
?>
_<?php
echo $outputNames[$index];
?>
= o._<?php
echo $outputNames[$index];
?>
;
<?php
}
// foreach value
?>
}
// Otherwise, count > 0 && o.count == 0, so just keep our values
count += o.count;
}
void GetResult(<?php
echo typed_ref_args($output);
?>
) {
<?php
foreach ($output as $k => $v) {
?>
<?php
echo $k;
?>
= _<?php
echo $k;
?>
;
<?php
}
// foreach output
?>
}
};
<?php
return ['kind' => 'GLA', 'name' => $name, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'system_headers' => ['algorithm', 'cstdint']];
}
/**
* Normal stuff we need to get ready before we do anything else
*
* @return boolean
* @access public
*/
function load()
{
$db =& $GLOBALS['SQ_SYSTEM']->db;
$GLOBALS['SQ_SYSTEM']->changeDatabaseConnection('db');
$dbtype = $this->_getDbType();
// load our tables from the database
switch ($dbtype) {
case 'oci':
$sql = "SELECT object_name AS table_name FROM user_objects WHERE object_type = 'TABLE'";
break;
default:
$sql = "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE'";
break;
}
//end switch
$results = MatrixDAL::executeSqlAssoc($sql);
$tables = array();
foreach ($results as $table) {
$table_name = array_get_index($table, 'table_name', '');
if (!empty($table_name)) {
$tables[] = strtolower($table_name);
}
//end if
}
//end foreach
$this->tables = $tables;
if (empty($this->tables)) {
return FALSE;
}
return TRUE;
}
function Segmenter(array $t_args, array $input, array $output, array $given_states)
{
$resType = ['fragment', 'multi'];
$system_headers = ['array', 'vector', 'memory', 'cinttypes', 'unordered_map'];
$user_headers = ['HashFunctions.h'];
$lib_headers = [];
$preferFragment = get_default($t_args, 'inner.prefer.fragment', false);
$wantedRes = $preferFragment ? ['fragment', 'multi'] : ['multi', 'fragment'];
$nInputs = \count($input);
grokit_assert($nInputs > 1, 'Segmenter: Not enough inputs specified!');
$keyName = array_keys($input)[0];
$keyType = array_get_index($input, 0);
$innerInputs = array_slice($input, 1, $nInputs - 1, true);
$gla = get_first_key($t_args, ['gla', 'GLA', 0]);
grokit_assert(is_gla($gla), 'Segmenter: [gla] argument must be a valid GLA');
$gla = $gla->apply($innerInputs, $output, $given_states);
$n_passes = get_default($t_args, 'passes', 1);
grokit_assert(is_int($n_passes), 'Segmenter: [passes] argument must be an integer');
grokit_assert($n_passes > 0, 'Segmenter: [passes] argument must be > 0');
$libraries = $gla->libraries();
$innerRes = get_first_value($gla->result_type(), $wantedRes);
$innerInputs = $gla->input();
$innerOutput = $gla->output();
$input = array_merge([$keyName => $keyType], $innerInputs);
$output = $innerOutput;
$segments = get_default($t_args, 'segments', 64);
$constState = lookupResource('BASE::SegmenterState', ['gla' => $gla, 'passes' => $n_passes, 'segments' => $segments]);
$className = generate_name('Segmenter_');
$savedArgs = [];
$cArgs = [];
$innerCArgs = [];
if ($gla->configurable()) {
$savedArgs['json_init'] = 'Json::Value';
$cArgs['json_init'] = 'Json::Value';
$innerCArgs[] = 'json_init';
}
$cArgs['const_state'] = $constState;
if ($gla->has_state()) {
$innerCArgs[] = 'constState.inner_cstate';
}
$cstStr = \count($innerCArgs) > 0 ? '(' . implode(',', $innerCArgs) . ')' : '';
grokit_assert(!$gla->iterable(), 'Segementer does not support iterable GLAs');
$iterable = $n_passes > 1;
?>
class <?php
echo $className;
?>
{
private:
using ConstantState = <?php
echo $constState;
?>
;
using SplitState = ConstantState::SplitState;
static constexpr const size_t NUM_STATES = SplitState::NUM_STATES;
using InnerGLA = <?php
echo $gla;
?>
;
using InnerGLAPtr = std::unique_ptr<InnerGLA>;
using GLA_Array = std::array<InnerGLAPtr, NUM_STATES>;
public:
using size_type = std::size_t;
<?php
if ($innerRes == 'fragment') {
?>
class Iterator {
private:
InnerGLA * gla;
int fragmentNum;
InnerGLA::Iterator * innerIter;
public:
Iterator( InnerGLA * _gla, int _fragmentNum, int _innerFrag ) :
gla(_gla), fragmentNum(_fragmentNum),
innerIter(nullptr)
{
innerIter = gla->Finalize(_innerFrag);
}
~Iterator(void) {
if( innerIter != nullptr ) {
delete innerIter;
innerIter = nullptr;
}
}
bool GetNextResult( <?php
echo typed_ref_args($gla->output());
?>
) {
return innerIter->GetNextResult(<?php
echo args($gla->output());
?>
);
}
int FragmentNumber() {
return fragmentNum;
}
};
<?php
} else {
// if inner result type is fragment
?>
class Iterator {
private:
InnerGLA * gla;
int fragmentNum;
public:
Iterator( InnerGLA * _gla, int fragNo ) : gla(_gla), fragmentNum(fragNo) {
gla->Finalize();
}
~Iterator(void) { }
bool GetNextResult( <?php
echo typed_ref_args($gla->output());
?>
) {
return gla->GetNextResult(<?php
echo args($gla->output());
?>
);
}
int FragmentNumber() {
return fragmentNum;
}
};
<?php
}
// if inner result type is multi
?>
private:
const ConstantState & constState;
GLA_Array localState;
// Iteration state for multi result type
int numFrags;
int multiFragNo;
Iterator * multiIter;
<?php
if ($innerRes == 'fragment') {
?>
using frag_info = std::pair<int, int>;
using frag_map_t = std::unordered_map<int, frag_info>;
frag_map_t fragMap;
<?php
}
?>
<?php
foreach ($savedArgs as $name => $type) {
?>
const <?php
echo $type;
?>
<?php
echo $name;
?>
;
<?php
}
// foreach saved arg
?>
public:
// Constructor
<?php
echo $className;
?>
( <?php
echo const_typed_ref_args($cArgs);
?>
) :
constState(const_state)
, localState()
, numFrags(0)
, multiFragNo(0)
, multiIter(nullptr)
<?php
if ($innerRes == 'fragment') {
?>
, fragMap()
<?php
}
foreach ($savedArgs as $name => $type) {
?>
, <?php
echo $name;
?>
(<?php
echo $name;
?>
)
<?php
}
// foreach constructor arg to save
?>
{
for( auto & elem : localState ) {
elem.reset(new InnerGLA<?php
echo $cstStr;
?>
);
}
}
void AddItem( <?php
echo const_typed_ref_args($input);
?>
) {
uint64_t hashVal = CongruentHash(Hash(<?php
echo $keyName;
?>
), H_b + 1);
uint64_t passNum = (hashVal / NUM_STATES) % ConstantState::N_PASSES;
uint64_t segNum = hashVal % NUM_STATES;
<?php
if ($n_passes > 1) {
?>
if( passNum != constState.pass ) {
return;
}
<?php
}
// more than 1 pass
?>
localState[segNum]->AddItem(<?php
echo args($innerInputs);
?>
);
}
void ChunkBoundary(void) {
// Merge local states into the global state
SplitState & globalStates = constState.segments;
int theseAreOk[NUM_STATES];
for( int i = 0; NUM_STATES > i; i++ ) {
theseAreOk[i] = 1;
}
int segsLeft = NUM_STATES;
while( segsLeft > 0 ) {
InnerGLA * checkedOut = nullptr;
int whichOne = globalStates.CheckOutOne( theseAreOk, checkedOut );
if( checkedOut == NULL ) {
checkedOut = new InnerGLA<?php
echo $cstStr;
?>
;
}
checkedOut->AddState( *(localState[whichOne]) );
globalStates.CheckIn( whichOne, checkedOut );
theseAreOk[whichOne] = 0;
segsLeft--;
}
// Re-initialize the local states
for( auto & elem : localState ) {
<?php
if ($gla->is('resettable')) {
?>
elem->Reset();
<?php
} else {
// if resettable
?>
elem.reset(new InnerGLA<?php
echo $cstStr;
?>
);
<?php
}
// if not resettable
?>
}
}
void AddState( <?php
echo $className;
?>
& o ) {
// Do nothing
}
void Finalize() {
SplitState & globalStates = constState.segments;
if( multiIter != nullptr)
delete multiIter;
multiFragNo = 0;
<?php
if ($innerRes == 'fragment') {
?>
frag_info fInfo = fragMap[multiFragNo];
multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo,
fInfo.second);
<?php
} else {
?>
multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo);
<?php
}
?>
}
bool GetNextResult(<?php
echo typed_ref_args($output);
?>
) {
bool gotResult = false;
SplitState & globalStates = constState.segments;
while( (multiFragNo < numFrags && multiIter != nullptr) && !gotResult ) {
gotResult = multiIter->GetNextResult(<?php
echo args($output);
?>
);
if( !gotResult ) {
multiFragNo++;
delete multiIter;
if( numFrags > multiFragNo ) {
<?php
if ($innerRes == 'fragment') {
?>
frag_info fInfo = fragMap[multiFragNo];
multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo,
fInfo.second);
<?php
} else {
?>
multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo);
<?php
}
?>
} else {
multiIter = nullptr;
}
}
}
return gotResult;
}
int GetNumFragments(void) {
<?php
if ($innerRes == 'fragment') {
?>
SplitState & globalStates = constState.segments;
numFrags = 0;
for (int i = 0; i < NUM_STATES; i++) {
int curFrags = globalStates.Peek(i)->GetNumFragments();
for (int curFrag = 0; curFrag < curFrags; curFrag++) {
fragMap[numFrags] = frag_info(i, curFrag);
numFrags++;
}
}
<?php
} else {
?>
numFrags = NUM_STATES;
<?php
}
?>
return numFrags;
}
Iterator * Finalize( int fragment ) {
SplitState & globalStates = constState.segments;
<?php
if ($innerRes == 'fragment') {
?>
frag_info info = fragMap[fragment];
return new Iterator(globalStates.Peek(info.first), fragment, info.second);
<?php
} else {
?>
return new Iterator(globalStates.Peek(fragment), fragment);
<?php
}
?>
}
bool GetNextResult( Iterator * it, <?php
echo typed_ref_args($output);
?>
) {
bool ret = it->GetNextResult(<?php
echo args($output);
?>
);
return ret;
}
<?php
if ($iterable) {
?>
bool ShouldIterate( ConstantState & modible ) {
modible.pass++;
return modible.pass < ConstantState::N_PASSES;
}
void PostFinalize() {
constState.segments.Reset();
}
<?php
}
// iterable
?>
<?php
if ($gla->is('finite container')) {
?>
size_type size() {
SplitState & globalStates = constState.segments;
size_type s = 0;
for( int i = 0; NUM_STATES > i; i++ ) {
InnerGLA * ptr = globalStates.Peek(i);
s += ptr->size();
}
return s;
}
size_type size(int frag) {
SplitState & globalStates = constState.segments;
return globalStates.Peek(frag)->size();
}
<?php
}
// if the gla is a container
?>
};
typedef <?php
echo $className;
?>
::Iterator <?php
echo $className;
?>
_Iterator;
<?php
return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $system_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'input' => $input, 'output' => $output, 'result_type' => $resType, 'generated_state' => $constState, 'required_states' => $gla->req_states(), 'chunk_boundary' => true, 'configurable' => $gla->configurable(), 'iterable' => $iterable, 'post_finalize' => $iterable, 'intermediates' => true];
}
/**
* A GLA that estimates the cardinality of a dataset using a bloom filter of
* a configurable size.
*
* Note: This filter has very high performance, so long as all of the states
* fit into cache, preferably L1 or L2, but L3 is also fine. Once the states
* are large enough that all of them cannot fit inside L3 cache at the same
* time, performance takes a nose dive (4x loss minimum).
*/
function BloomFilter(array $t_args, array $input, array $output)
{
grokit_assert(\count($output) == 1, 'BloomFilter produces only 1 value, ' . \count($output) . ' outputs given.');
$outputName = array_keys($output)[0];
$outputType = array_get_index($output, 0);
if (is_null($outputType)) {
$outputType = lookupType('BASE::BIGINT');
}
$output[$outputName] = $outputType;
grokit_assert($outputType->is('numeric'), 'BloomFilter output must be numeric!');
$exp = get_first_key_default($t_args, ['exponent'], 16);
grokit_assert(is_integer($exp), 'BloomFilter exponent must be an integer.');
grokit_assert($exp > 0 && $exp < 64, 'BloomFilter exponent must be in range (0,64), ' . $exp . ' given.');
$nullCheck = get_default($t_args, 'null.check', false);
$nullable = [];
if (is_bool($nullCheck)) {
foreach ($input as $name => $type) {
$nullable[$name] = $nullCheck;
}
} else {
if (is_array($nullCheck)) {
foreach ($input as $name => $type) {
$nullable[$name] = false;
}
foreach ($nullCheck as $index => $n) {
grokit_assert(is_string($n), 'BloomFilster null.check has invalid value at position ' . $index);
grokit_assert(array_key_exists($n, $nullable), 'BloomFilster null.check has unknown input ' . $n . ' at position ' . $index);
$nullable[$n] = true;
}
} else {
grokit_error('BloomFilster null.check must be boolean or list of inputs to check for nulls');
}
}
$debug = get_default($t_args, 'debug', 0);
$bits = pow(2, $exp);
$bytes = ceil($bits / 8.0);
// Calculate the number of bits set for every possible value of a byte
$nBits = [];
for ($i = 0; $i < 256; $i++) {
$n = $i;
$b = 0;
while ($n > 0) {
$n &= $n - 1;
$b++;
}
$nBits[$i] = $b;
}
$className = generate_name('BloomFilter');
?>
class <?php
echo $className;
?>
{
static constexpr size_t BITS = <?php
echo $bits;
?>
;
static constexpr size_t BYTES = <?php
echo $bytes;
?>
;
static constexpr size_t MASK = BITS - 1;
static constexpr std::array<unsigned char, 256> BITS_SET = { <?php
echo implode(', ', $nBits);
?>
};
static constexpr std::array<unsigned char, 8> BIT_MASKS = {
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80
};
size_t count;
std::array<unsigned char, BYTES> set;
//unsigned char set[BYTES];
//std::bitset<BITS> set;
public:
<?php
echo $className;
?>
() : count(0), set() {
for( size_t i = 0; i < BYTES; i++ ) { //>
set[i] = 0;
}
}
~<?php
echo $className;
?>
() { }
void AddItem( <?php
echo const_typed_ref_args($input);
?>
) {
count++;
<?php
foreach ($nullable as $name => $check) {
if ($check) {
?>
if( IsNull( <?php
echo $name;
?>
) ) return;
<?php
}
// if checking for nulls
}
// foreach input
?>
size_t hashVal = H_b;
<?php
foreach ($input as $name => $type) {
?>
hashVal = CongruentHash(Hash(<?php
echo $name;
?>
), hashVal);
<?php
}
// foreach input
?>
hashVal = hashVal & MASK;
const size_t bucket = hashVal >> 3;
const size_t bucket_index = hashVal & 0x07;
const unsigned char mask = BIT_MASKS[bucket_index];
set[bucket] |= mask;
}
void AddState( <?php
echo $className;
?>
& o ) {
count += o.count;
for( size_t i = 0; i < BYTES; i++ ) { //>
set[i] |= o.set[i];
}
}
void GetResult( <?php
echo $outputType;
?>
& <?php
echo $outputName;
?>
) {
size_t nBitsSet = 0;
constexpr long double bits = static_cast<long double>(BITS);
for( size_t i = 0; i < BYTES; i++ ) { //>
nBitsSet += BITS_SET[set[i]];
}
long double bitsSet = static_cast<long double>(nBitsSet);
if( nBitsSet == BITS ) {
// All Bits set, just give the cardinality as an estimate.
<?php
echo $outputName;
?>
= count;
} else {
long double cardinality = - bits * std::log(1 - (bitsSet / bits));
<?php
echo $outputName;
?>
= cardinality;
}
<?php
if ($debug > 0) {
?>
std::cout << "BloomFilter:"
<< " bitsSet(" << bitsSet << ")"
<< " bits(" << bits << ")"
<< " cardinality(" << cardinality << ")"
<< " output(" << <?php
echo $outputName;
?>
<< ")"
<< std::endl;; //>
<?php
}
// if debugging enabled
?>
}
};
// Storage for static members
constexpr std::array<unsigned char, 256> <?php
echo $className;
?>
::BITS_SET;
constexpr std::array<unsigned char, 8> <?php
echo $className;
?>
::BIT_MASKS;
<?php
$system_headers = ['cmath', 'array'];
if ($debug > 0) {
$system_headers[] = 'iostream';
}
return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'user_headers' => ['HashFunctions.h'], 'system_headers' => $system_headers];
}
* report invalid new url remaps
*/
if (getCLIArg('invalid_new_url')) {
echo "\n## Invalid New URL ##\n";
$sql = 'SELECT * FROM sq_ast_lookup_remap WHERE expires > :time OR expires is null';
$query = MatrixDAL::preparePdoQuery($sql);
$time = ts_iso8601(time());
MatrixDAL::bindValueToPdo($query, 'time', $time);
$result = MatrixDAL::executePdoAll($query);
$affected_result = array();
$root_urls = explode("\n", SQ_CONF_SYSTEM_ROOT_URLS);
foreach ($result as $index => $url_info) {
$new_url_parts = parse_url($url_info['remap_url']);
$protocol = array_get_index($new_url_parts, 'scheme', NULL);
if (!empty($protocol)) {
$url = array_get_index($new_url_parts, 'host', '') . array_get_index($new_url_parts, 'path', '');
// does it match with root matrix url
$is_matrix_url = FALSE;
foreach ($root_urls as $root_url) {
if (strpos($url, $root_url . '/') === 0 || $url === $root_url) {
$is_matrix_url = TRUE;
break;
}
}
// if it's not a Matrix URL, no need to show warning
if ($is_matrix_url) {
// if it's one of special url, no need to show warning
if (strpos($url, '__data') === FALSE && strpos($url, '__lib') === FALSE && strpos($url, '__fudge') === FALSE) {
$url_asset = $GLOBALS['SQ_SYSTEM']->am->getAssetFromURL($protocol, $url, TRUE, TRUE);
$count_query = 'SELECT count(*) FROM sq_ast_lookup_remap WHERE url = :url ';
$count_result = MatrixDAL::preparePdoQuery($count_query);
}
if (!is_dir($SYSTEM_ROOT) || !is_readable($SYSTEM_ROOT . '/core/include/init.inc')) {
echo "ERROR: Path provided doesn't point to a Matrix installation's System Root. Please provide correct path and try again.\n";
echo 'Usage: ' . basename(__FILE__) . " SYSTEM_ROOT [NEW_PASSWORD]\n\n";
echo " If NEW_PASSWORD is not provided, it will be reset to 'root'\n";
exit;
}
require_once $SYSTEM_ROOT . '/core/include/init.inc';
// Get the root user
$root_user =& $GLOBALS['SQ_SYSTEM']->am->getSystemAsset('root_user');
// log in as root :P
if (!$GLOBALS['SQ_SYSTEM']->setCurrentUser($root_user)) {
echo "ERROR: Failed login in as root user\n";
exit;
}
// try to lock the root user
if (!$GLOBALS['SQ_SYSTEM']->am->acquireLock($root_user->id, 'attributes')) {
trigger_error("Couldn't get lock\n", E_USER_ERROR);
}
$current_run_level = $GLOBALS['SQ_SYSTEM']->getRunLevel();
$GLOBALS['SQ_SYSTEM']->setRunLevel($current_run_level - SQ_SECURITY_PASSWORD_VALIDATION);
$password = array_get_index($argv, 2, 'root');
if (!$root_user->setAttrValue('password', $password)) {
trigger_error("Couldn't set password\n", E_USER_ERROR);
}
if (!$root_user->saveAttributes()) {
trigger_error("Couldn't save attributes \n", E_USER_ERROR);
}
$GLOBALS['SQ_SYSTEM']->restoreRunLevel();
$GLOBALS['SQ_SYSTEM']->am->releaseLock($root_user->id, 'attributes');
echo 'Root User Password now reset to "' . $password . '", please login and change.', "\n";
/**
* Fixes the char encoding in the given tables in the database
*
* @param int $root_node Assetid of rootnode, all childern of rootnode will be processed for char replacement
* @param array $tables DB tables and colunms info
* @param boolean $rollback If TRUE process rollback tables, else process regular tables
*
* @return void
*/
function fix_db($root_node, $tables, $rollback)
{
global $reportOnly;
$tables_info = get_tables_info();
// All the Matrix attribute types with serialised value
$serialsed_attrs = array('option_list', 'email_format', 'parameter_map', 'serialise', 'http_request', 'oauth');
// Get the list of attrids of the type 'serialise'
$sql = "SELECT attrid FROM sq_ast_attr WHERE type IN ('" . implode("','", $serialsed_attrs) . "')";
$serialise_attrids = array_keys(MatrixDAL::executeSqlGrouped($sql));
if ($root_node != 1) {
// Get the targetted asset list
$target_assetids = array_keys($GLOBALS['SQ_SYSTEM']->am->getChildren($root_node));
// Since we include the root node, target assetids will always contain atleast one asset id
array_unshift($target_assetids, $root_node);
echo "\n\nNumber of assets to look into : " . count($target_assetids) . " \n";
// Go through 50 assets at a time. Applicable to asset specific tables only
$chunks = array_chunk($target_assetids, 50);
$chunks_count = count($chunks);
}
$errors_count = 0;
$warnings_count = 0;
$records_fixed_count = 0;
$invalid_asset_records = array();
// Assets that will require filesystem content regeneration
$affected_assetids = array();
$GLOBALS['SQ_SYSTEM']->changeDatabaseConnection('db2');
// Counter to count the number of records accessed/processed
$count = 0;
foreach ($tables as $table_data) {
$table_records_count = 0;
$table = isset($table_data['table']) ? $table_data['table'] : '';
if (empty($table)) {
continue;
}
$key_fields = isset($tables_info[$table]['primary_key']) ? $tables_info[$table]['primary_key'] : '';
if (empty($key_fields)) {
echo "\n" . 'Ignoring table "' . $table . '". Table info for this table not found' . " \n";
continue;
}
$value_fields = isset($table_data['values']) ? $table_data['values'] : '';
if (empty($value_fields)) {
// Nothing to check
continue;
}
if ($rollback) {
// Make sure table has rollback trigggers enabled, otherwise it will have rollback table
if (isset($tables_info[$table]['rollback']) && $tables_info[$table]['rollback']) {
// Add rollback table primary key field to the table's keys
$key_fields[] = 'sq_eff_from';
} else {
// This table does not has corresponding rollback table
continue;
}
}
// Prepend table prefix
$table = !$rollback ? 'sq_' . $table : 'sq_rb_' . $table;
$asste_specific_table = $table_data['asset_assoc'];
$select_fields = array_merge($value_fields, $key_fields);
if ($asste_specific_table && !in_array('assetid', $select_fields)) {
$select_fields[] = 'assetid';
}
if ($root_node == 1) {
if ($asste_specific_table) {
// When running system wide, get the asset list from the respective db table
$sql = "SELECT DISTINCT assetid FROM " . $table;
$target_assetids = array_keys(MatrixDAL::executeSqlGrouped($sql));
// Go through 50 assets at a time. Applicable to asset specific tables only
$chunks = array_chunk($target_assetids, 50);
} else {
// Dummy assetids chuck just so that we can get into next loop
$chunks = array(array());
}
}
echo "\nChecking " . $table . " .";
// For non-asset specific table, this loop will break at end of the very first iteration
foreach ($chunks as $chunk_index => $assetids) {
$sql = 'SELECT ' . implode(',', $select_fields) . ' FROM ' . $table;
// For non-asset specific table, "where" condition not is required. We get the whole table in a single go
if ($asste_specific_table) {
$sql .= ' WHERE assetid IN (\'' . implode('\',\'', $assetids) . '\')';
} else {
if ($table == 'sq_internal_msg') {
// Special case for non-asset specific records for 'interal_msg' table
// Internal message has 'assetid' field but messages not associated with the asset will have empty assetid
$sql .= " WHERE assetid = '' OR assetid IS NULL";
}
}
$results = MatrixDAL::executeSqlAssoc($sql);
foreach ($results as $record) {
$table_records_count++;
$count++;
if ($count % 10000 == 0) {
echo '.';
}
// Asset ID associated with this record
$assetid = $asste_specific_table ? $record['assetid'] : 'n/a';
// Key field data
$key_values = array();
foreach ($key_fields as $key_field) {
$temp_key_v = array_get_index($record, $key_field, NULL);
if (is_null($temp_key_v)) {
// Primary key field must be there
continue 2;
}
$key_values[$key_field] = $temp_key_v;
}
//end foreach
// Original value field data.
// This is the one we need to check/fix
$org_values = array();
foreach ($value_fields as $value_field) {
$org_values[$value_field] = array_get_index($record, $value_field, '');
}
//end foreach
// If it's the same in the new and old encodings, that's good.
foreach ($org_values as $value_field => $value) {
$checked = @iconv(SYS_OLD_ENCODING, SYS_NEW_ENCODING . '//IGNORE', $value);
if ($value === $checked) {
// This field does not requires conversion/checking
unset($org_values[$value_field]);
}
}
//end foreach
if (empty($org_values)) {
// No field values to convert/check
continue;
}
// Being here means this record contains invalid chars
$invalid_asset_records[] = array('asset' => $assetid, 'table' => $table, 'keys' => $key_values, 'values' => $org_values);
$converted_values = array();
foreach ($org_values as $value_field => $value) {
// If not valid, convert the values without igonoring or interprating any chars
if (!isValidValue($value)) {
// Serialised fields needs to be handled here
$serialised_value = FALSE;
if ($table == 'sq_ast_attr_val' && $value_field == 'custom_val' && in_array($record['attrid'], $serialise_attrids)) {
$serialised_value = TRUE;
}
if ($table == 'sq_trig' && $value_field == 'data') {
$serialised_value = TRUE;
}
if ($serialised_value) {
$us_value = @unserialize($value);
if ($us_value === FALSE && serialize(FALSE) !== $value) {
// This has invalid serialsed value, but fix it anyway
$converted_value = @iconv(SYS_OLD_ENCODING, SYS_NEW_ENCODING . '//IGNORE', $value);
// Put this error notice in the script log file
$warnings_count++;
$msg = 'Serialsed data field "' . $value_field . '" in the table "' . $table . '" (';
foreach ($key_values as $field_name => $value) {
$msg .= $field_name . '=' . $value . '; ';
}
$msg = rtrim($msg, '; ') . ') does not contain unserialisable data. ' . ($reportOnly ? 'Data can still be converted.' : 'Data will be converted anyway.');
log_error_msg($msg);
} else {
if (is_array($us_value)) {
array_walk_recursive($us_value, 'fix_char');
$converted_value = serialize($us_value);
} else {
if (is_scalar($us_value)) {
$us_value = @iconv(SYS_OLD_ENCODING, SYS_NEW_ENCODING . '//IGNORE', $us_value);
$converted_value = serialize($us_value);
} else {
$converted_value = $value;
}
}
}
} else {
$converted_value = @iconv(SYS_OLD_ENCODING, SYS_NEW_ENCODING . '//IGNORE', $value);
}
// If the converted value is valid in current encoding then its good to go
// otherwise we'll just not use this value
if ($converted_value != $value && isValidValue($converted_value)) {
$value = $converted_value;
$converted_values[$value_field] = $value;
}
} else {
// if it's a valid encoded value, but was convertable before with iconv using old encoding
// it might be only because value is already properly encoded with new encoding. so use md_detect to double check
$encoding = mb_detect_encoding($value);
if (strtolower($encoding) === strtolower(SYS_NEW_ENCODING)) {
unset($org_values[$value_field]);
}
}
}
//end foreach
if (empty($org_values)) {
// All good
array_pop($invalid_asset_records);
continue;
}
// If the successfully converted fields count is same as the invalid fields count, we can proceed with the update
$update_required = count($org_values) == count($converted_values);
if ($update_required) {
if (!$reportOnly) {
$GLOBALS['SQ_SYSTEM']->doTransaction('BEGIN');
// Generate update sql
$bind_vars = array();
$set_sql = array();
foreach ($converted_values as $field_name => $value) {
$set_sql[] = $field_name . '=:' . $field_name . '_v';
$bind_vars[$field_name . '_v'] = $value;
}
$where_sql = array();
foreach ($key_values as $field_name => $value) {
$where_sql[] = $field_name . '=:' . $field_name . '_k';
$bind_vars[$field_name . '_k'] = $value;
}
try {
$sql = 'UPDATE ' . $table . '
SET ' . implode(', ', $set_sql) . '
WHERE ' . implode(' AND ', $where_sql);
$update_sql = MatrixDAL::preparePdoQuery($sql);
foreach ($bind_vars as $var_name => $var_value) {
MatrixDAL::bindValueToPdo($update_sql, $var_name, $var_value);
}
// Execute the update query
$execute = MatrixDAL::executePdoAssoc($update_sql);
if (count($execute) > 1) {
foreach ($bind_vars as $var_name => $var_value) {
$sql = str_replace(':' . $var_name, "'" . $var_value . "'", $sql);
}
$errors_count++;
$msg = 'Executing query "' . $sql . '" will affect ' . count($execute) . ' records, instead of expected single record! Ignoring this sql.';
log_error_msg($msg);
$GLOBALS['SQ_SYSTEM']->doTransaction('ROLLBACK');
} else {
$GLOBALS['SQ_SYSTEM']->doTransaction('COMMIT');
$records_fixed_count++;
$affected_assetids[$table][] = $assetid;
}
} catch (Exception $e) {
$errors_count++;
$msg = "Unexpected error occured while updating database: " . $e->getMessage();
log_error_msg($msg);
$GLOBALS['SQ_SYSTEM']->doTransaction('ROLLBACK');
}
} else {
$records_fixed_count++;
// For reporting purpose only
$affected_assetids[$table][] = $assetid;
}
} else {
// Trying to carryout charset conversion for this invalid value still resulted into invalid value
// Hence record was not updated for this value conversion
$errors_count++;
$msg = 'Entry in the table "' . $table . '": ' . "\n";
foreach ($key_values as $field_name => $field_value) {
$msg .= $field_name . '="' . $field_value . '"; ';
}
$msg .= "\n" . 'contains invalid char(s), which were not replaced because the charset conversion was not successful' . ($msg .= "\n" . 'Potentially invalid characters include:' . listProblematicCharacters($org_values));
log_error_msg($msg);
}
}
//end foreach records
if (!$asste_specific_table) {
// We have processed all the entries for this non-asset specific table
break;
}
}
//end foreach assetids chunk
echo " " . $table_records_count . " records";
}
//end foreach tables
$GLOBALS['SQ_SYSTEM']->restoreDatabaseConnection();
unset($target_assetids);
unset($chunks);
echo "\n";
$invalid_count = sizeof(array_keys($invalid_asset_records));
echo "Number of db records with invalid char(s): " . $invalid_count . "\n";
if ($invalid_count > 0) {
foreach ($invalid_asset_records as $k => $details) {
echo "\n\tAsset #" . $details['asset'] . " in table " . $details['table'];
echo "\n\t" . 'Entry: ';
foreach ($details['keys'] as $field_name => $field_value) {
echo $field_name . '="' . $field_value . '"; ';
}
echo "\n\tPossibly problematic characters: " . listProblematicCharacters($details['values']) . "\n";
}
echo "\n";
}
return array('warning_count' => $warnings_count, 'error_count' => $errors_count, 'records_fixed_count' => $records_fixed_count, 'affected_assetids' => $affected_assetids);
}
