コード例 #1
0
function PatternMatcherOnig($t_args, $inputs)
{
    grokit_assert(\count($inputs) == 1, 'PatternMatcherOnig GF only supports 1 input!');
    $pattern = get_first_key($t_args, ['pattern']);
    $inName = array_keys($inputs)[0];
    $inType = array_get_index($inputs, 0);
    $inTypeString = $inType->name();
    $validTypes = ['BASE::STRING_LITERAL'];
    grokit_assert(in_array($inTypeString, $validTypes), 'Unsupported input type ' . $inTypeString);
    $className = generate_name('PatternMatcherOnigGF');
    ?>

class <?php 
    echo $className;
    ?>
 {

    PatternMatcherOnig matcher;

public:
    <?php 
    echo $className;
    ?>
() :
        matcher("<?php 
    echo $pattern;
    ?>
")
    { }

    bool Filter( const <?php 
    echo $inType;
    ?>
 & <?php 
    echo $inName;
    ?>
 ) {
        return matcher.Match(<?php 
    echo $inName;
    ?>
);
    }
};

<?php 
    return ['kind' => 'GF', 'name' => $className, 'input' => $inputs, 'user_headers' => ['PatternMatcherOnig.h']];
}
コード例 #2
0
* | This MySource Matrix CMS file is Copyright (c) Squiz Pty Ltd       |
* | ABN 77 084 670 600                                                 |
* +--------------------------------------------------------------------+
* | IMPORTANT: Your use of this Software is subject to the terms of    |
* | the Licence provided in the file licence.txt. If you cannot find   |
* | this file please contact Squiz (www.squiz.com.au) so we may provide|
* | you a copy.                                                        |
* +--------------------------------------------------------------------+
*
* $Id: session.php,v 1.9 2012/08/30 01:09:21 ewang Exp $
*
*/
// include init.in, which will call $GLOBALS['SQ_SYSTEM']->init() and start a session
require_once '../../include/init.inc';
header("content-type: application/javascript");
$site_network_id = array_get_index($_GET, 'site_network', '0');
$site_network = null;
$primary_url = '';
$session_handler = $GLOBALS['SQ_SYSTEM']->getSessionHandlerClassName();
if ($site_network_id) {
    $site_network =& $GLOBALS['SQ_SYSTEM']->am->getAsset($site_network_id);
    if (!is_null($site_network)) {
        $primary_url = $site_network->getPrimaryURL();
    }
}
if ($primary_url == sq_web_path('root_url')) {
    // if the actual script execution is happening in the primary
    // url, we want to update the time to persuade the overriding
    // of the secondary session files
    if (isset($_GET['in_primary']) && $_GET['in_primary']) {
        $_SESSION['SQ_SESSION_TIMESTAMP'] = time();
コード例 #3
0
//end if
// Check for a lock file
if (file_exists(SQ_TEMP_PATH . '/funnelback.indexer')) {
    trigger_localised_error('FNB0019', translate('Lockfile found. Scripts cannot be run concurrently.'), E_USER_WARNING);
    exit;
}
//end if
// Create a lock file
touch(SQ_TEMP_PATH . '/funnelback.indexer');
// Start indexing
$hh = $GLOBALS['SQ_SYSTEM']->getHipoHerder();
$vars = array('collections' => empty($root_collection) ? array() : $root_collection);
$errors = $hh->freestyleHipo('hipo_job_funnelback_reindex', $vars, SQ_PACKAGES_PATH . '/funnelback/hipo_jobs');
if (!empty($errors)) {
    echo 'Funnelback Reindexing FAILED' . "\n";
    foreach ($errors as $error) {
        $line = array_get_index($error, 'message', '');
        if (!empty($line)) {
            echo $line . "\n";
        }
        //end if
    }
    //end foreach
}
//end if
// Remove if finished AND failed
if (file_exists(SQ_TEMP_PATH . '/funnelback.indexer')) {
    unlink(SQ_TEMP_PATH . '/funnelback.indexer');
}
//end if
$GLOBALS['SQ_SYSTEM']->restoreRunLevel();
コード例 #4
0
if (php_sapi_name() != 'cli') {
    trigger_error('This script can only be run from the command line', E_USER_ERROR);
    exit;
}
//end if
$SYSTEM_ROOT = isset($_SERVER['argv'][1]) ? $_SERVER['argv'][1] : '';
if (empty($SYSTEM_ROOT) || !is_readable($SYSTEM_ROOT)) {
    trigger_error('You need to specify a path to Matrix as the first parameter', E_USER_ERROR);
    exit;
}
//end if
define('SQ_SYSTEM_ROOT', $SYSTEM_ROOT);
require_once SQ_SYSTEM_ROOT . '/core/include/init.inc';
// Check the script is being run from same user as apache
$apache_user = fileowner(SQ_SYSTEM_ROOT . '/data/private/conf/main.inc');
$current_user = array_get_index($_ENV, 'USER', '');
if ($apache_user != $current_user) {
    //	trigger_error('This script needs to run as the apache user', E_USER_ERROR);
    //	exit();
}
//end if
$funnelback_manager = $GLOBALS['SQ_SYSTEM']->am->getSystemAsset('funnelback_manager');
// Check the base install for paths and permissions
$valid = $funnelback_manager->checkInstalled();
if ($valid) {
    echo "Looks ok from here\n";
} else {
    echo "Funnelback has not been installed correctly\n";
}
//end if
exit;
コード例 #5
0
ファイル: HyperLogLog.h.php プロジェクト: gokulp/grokit
/**
 *  A GLA that estimates the cardinality of a dataset using the HyperLogLog
 *  algorithm, with a configurable number of bins.
 */
function HyperLogLog(array $t_args, array $input, array $output)
{
    $debug = get_default($t_args, 'debug', 0);
    grokit_assert(\count($output) == 1, 'HyperLogLog produces only 1 value, ' . \count($output) . ' outputs given.');
    $outputName = array_keys($output)[0];
    $outputType = array_get_index($output, 0);
    if (is_null($outputType)) {
        $outputType = lookupType('BASE::BIGINT');
    }
    $output[$outputName] = $outputType;
    grokit_assert($outputType->is('numeric'), 'BloomFilter output must be numeric!');
    $exp = get_first_key_default($t_args, ['bins.exponent'], 4);
    grokit_assert(is_integer($exp), 'HyperLogLog bins.exponent must be an integer');
    // Set limit of 2^24 bins, because states past 16MB start to get silly
    grokit_assert($exp >= 4 && $exp < 24, 'HyperLogLog bins.exponent must be in range [4, 24]');
    $useBuiltinCtz = get_default($t_args, 'use.builtin.ctz', true);
    $ctzFunc = $useBuiltinCtz ? '__builtin_ctzl' : 'ctz';
    $bins = pow(2, $exp);
    // Determine the value of alpha based on $exp
    switch ($exp) {
        case 4:
            $alpha = 0.673;
            break;
        case 5:
            $alpha = 0.697;
            break;
        case 6:
            $alpha = 0.709;
            break;
        default:
            $alpha = 0.7213000000000001 / (1 + 1.079 / $bins);
    }
    $className = generate_name('HyperLogLog');
    ?>

class <?php 
    echo $className;
    ?>
 {
    // Number of bins for registers
    static constexpr const size_t NUM_BINS = <?php 
    echo $bins;
    ?>
;
    // Number of bits used to index into registers, log2(NUM_BINS)
    static constexpr const size_t INDEX_BITS = <?php 
    echo $exp;
    ?>
;
    // Mask used to obtain register index from hash value
    static constexpr const size_t INDEX_MASK = NUM_BINS - 1;

    // Alpha coefficient used to correct cardinality estimate. Based on NUM_BINS.
    static constexpr const long double ALPHA = <?php 
    echo $alpha;
    ?>
;

    // Value of cardinality estimate after which we must apply the
    // large range correction
    static constexpr const long double LARGE_BREAKPOINT = (1.0 / 30.0) * <?php 
    echo pow(2, 32);
    ?>
;

    // Constants for population count
    static constexpr const uint64_t m1  = 0x5555555555555555;
    static constexpr const uint64_t m2  = 0x3333333333333333;
    static constexpr const uint64_t m4  = 0x0f0f0f0f0f0f0f0f;
    static constexpr const uint64_t h01 = 0x0101010101010101;

    // The registers
    std::array<unsigned char, NUM_BINS> registers;

    // A count used to remember how many tuples were processed, mostly for debugging.
    size_t count;

public:

    <?php 
    echo $className;
    ?>
(void) : registers() {
        for( auto & elem : registers ) {
            elem = 0;
        }
    }

    ~<?php 
    echo $className;
    ?>
() { }

    int popcount(uint64_t x) {
        // Put count of each 2 bits into those 2 bits
        x -= (x >> 1) & m1;
        // Put count of each 4 bits into those 4 bits
        x = (x & m2) + ((x >> 2) & m2);
        // Put count of each 8 bits into those 8 bits
        x  = (x + (x >> 4)) & m4;
        // Returns left 8 bits of x + (x << 8) + (x << 16) + ...
        return (x * h01) >> 56;
    }

    int ctz(int64_t x) {
        return popcount((x & -x) - 1);
    }

    void AddItem( <?php 
    echo const_typed_ref_args($input);
    ?>
 ) {
        count++;

        uint64_t hashVal = H_b;
<?php 
    foreach ($input as $name => $type) {
        ?>
        hashVal = CongruentHash(Hash(<?php 
        echo $name;
        ?>
), hashVal);
<?php 
    }
    // for each input
    ?>

        const size_t registerIndex = hashVal & INDEX_MASK;
        uint64_t value = hashVal >> INDEX_BITS;
        unsigned char nZeros = <?php 
    echo $ctzFunc;
    ?>
(value);

        unsigned char & registerValue = registers[registerIndex];
        registerValue = registerValue > nZeros ? registerValue : nZeros;
    }

    void AddState( <?php 
    echo $className;
    ?>
 & other ) {
        for( size_t i = 0; NUM_BINS > i; i++ ) {
            unsigned char & rVal = registers[i];
            unsigned char & oVal = other.registers[i];
            rVal = rVal > oVal ? rVal : oVal;
        }
    }

    void GetResult( <?php 
    echo $outputType;
    ?>
 & <?php 
    echo $outputName;
    ?>
 ) {
        // Compute harmonic sum of registers and correct by alpha
        long double cardEst = 0;
        size_t nZeroRegisters = 0;
        for( auto elem : registers ) {
            long double power = - static_cast<long double>(elem);
            cardEst += std::pow(2.0, power);

            if( elem == 0 )
                nZeroRegisters++;
        }
        const long double nBins = static_cast<long double>(NUM_BINS);
        const long double zeroBins = static_cast<long double>(nZeroRegisters);

        cardEst = 1 / cardEst;
        cardEst *= ALPHA * nBins * nBins;

        long double cardinality = cardEst;

        if( (cardEst < 2.5 * NUM_BINS) ) { //>
            // Possible small range correction
            if( nZeroRegisters > 0 ) {
                // Small range correction
                cardinality = nBins * std::log(nBins / zeroBins);
            }
        }
        // TODO: Figure out if the large range correction is needed for 64-bit
        // hashes.

        <?php 
    echo $outputName;
    ?>
 = cardinality;
    }
};
<?php 
    $system_headers = ['cmath', 'array', 'cinttypes'];
    if ($debug > 0) {
        $system_headers[] = 'iostream';
    }
    return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'user_headers' => ['HashFunctions.h'], 'system_headers' => $system_headers];
}
コード例 #6
0
ファイル: grokit_typeinfo.php プロジェクト: gokulp/grokit
 public function __construct($kind, $hash, $name, $value, array $args, array $targs)
 {
     parent::__construct($kind, $hash, $name, $value, $args, $targs);
     if (array_key_exists('required_states', $args)) {
         $this->req_states = $args['required_states'];
     }
     if (array_key_exists('generated_state', $args)) {
         $this->gen_state = $args['generated_state'];
     }
     $this->configurable = get_default($args, 'configurable', false);
     if (array_key_exists('generated_states', $args)) {
         grokit_warning('"generated_states" type information parameter is deprecated. Please use "generated_state" instead.');
         $this->gen_states = ensure_unique_names($args['generated_states'], 'gen_state');
         if (is_null($this->gen_state) && \count($this->gen_states) == 1) {
             $this->gen_state = array_get_index($this->gen_states, 0);
         }
     }
     if (array_key_exists('constructor_args', $args)) {
         grokit_warning('"constructor_args" type information parameter is deprecated.');
         $this->constructor_args = ensure_unique_names($args['constructor_args'], 'const_arg');
     }
 }
コード例 #7
0
*
* $Id: keywords.php,v 1.1 2013/09/24 01:10:35 ewang Exp $
*
*/
require_once dirname(__FILE__) . '/../../../../core/include/init.inc';
require_once dirname(__FILE__) . '/../../../../core/lib/html_form/html_form.inc';
$assetid = array_get_index($_GET, 'assetid');
if (is_null($assetid)) {
    return FALSE;
}
assert_valid_assetid($assetid);
$asset = $GLOBALS['SQ_SYSTEM']->am->getAsset($assetid);
if (is_null($asset) || !$asset->writeAccess()) {
    exit;
}
$all = array_get_index($_REQUEST, 'all', FALSE);
require_once dirname(__FILE__) . '/../../../../core/include/backend_outputter.inc';
$o = new Backend_Outputter();
$o->openSection('Metadata Keyword Replacements');
$o->openField('', 'wide_col');
?>

		<p>The following keyword replacements may be used for metadata values. Note that the percentage signs (%) are required.</p>

		<?php 
$o->openSection('Asset Information');
$o->openField('', 'wide_col');
?>

			<table class="sq-backend-table compact">
			<tr><th style="width: 250px;">Keyword</th><th>Description</th></tr>
コード例 #8
0
$PERMISSION = isset($_SERVER['argv'][4]) ? $_SERVER['argv'][4] : '';
if (empty($PERMISSION) || !array_key_exists($PERMISSION, $available_permissions)) {
    echo "ERROR: You need to supply a permission as the fourth argument\n";
    exit;
}
$GRANTED = isset($_SERVER['argv'][5]) && $_SERVER['argv'][5] == 'y' ? '1' : '0';
$CHILDREN = isset($_SERVER['argv'][6]) && $_SERVER['argv'][6] == 'y' ? TRUE : FALSE;
// Start the process
$root_user =& $GLOBALS['SQ_SYSTEM']->am->getSystemAsset('root_user');
$GLOBALS['SQ_SYSTEM']->setCurrentUser($root_user);
$GLOBALS['SQ_SYSTEM']->setRunLevel(SQ_RUN_LEVEL_FORCED);
echo 'START PERMISSION CHANGE' . "\n";
$hh = $GLOBALS['SQ_SYSTEM']->getHipoHerder();
$vars = array('permission_changes' => array(array('permission' => $available_permissions[$PERMISSION], 'granted' => $GRANTED, 'assetids' => array($ROOT_NODE), 'userid' => $USER, 'previous_access' => NULL, 'cascades' => $CHILDREN)));
$errors = $hh->freestyleHipo('hipo_job_edit_permissions', $vars);
echo 'FINISHED';
if (!empty($errors)) {
    echo "... But with errors!\n";
    foreach ($errors as $error) {
        echo "\t" . ($line = array_get_index($error, 'message', ''));
        if (!empty($line)) {
            echo "\t" . $line . "\n";
        }
        //end if
    }
    //end foreach
} else {
    echo "\n";
}
//end if
$GLOBALS['SQ_SYSTEM']->restoreRunLevel();
コード例 #9
0
*
*/
/**
* Search and replaces given text in the WYSIWYG
*
* @author	Chiranjivi Upreti <*****@*****.**>
* @version 	$Revision: 1.5 $
* @package 	MySource_Matrix
*/
require_once dirname(__FILE__) . '/../../../../core/include/init.inc';
require_once SQ_LIB_PATH . '/html_form/html_form.inc';
require_once SQ_FUDGE_PATH . '/var_serialise/var_serialise.inc';
if (empty($GLOBALS['SQ_SYSTEM']->user) || !($GLOBALS['SQ_SYSTEM']->user->canAccessBackend() || $GLOBALS['SQ_SYSTEM']->user->type() == 'simple_edit_user' || method_exists($GLOBALS['SQ_SYSTEM']->user, 'isShadowSimpleEditUser') && $GLOBALS['SQ_SYSTEM']->user->isShadowSimpleEditUser())) {
    exit;
}
$_GET['name'] = array_get_index($_GET, 'name', '');
?>
<!doctype html>
<html style="height: 280px;">

	<head>
		<title>Search and Replace</title>
		<link rel="stylesheet" type="text/css" href="<?php 
echo sq_web_path('lib') . '/web/css/edit.css';
?>
" />
		<link rel="stylesheet" type="text/css" href="<?php 
echo sq_web_path('root_url');
?>
/__fudge/wysiwyg/core/popup.css" />
		<script type="text/javascript" src="../../core/popup.js"></script>
コード例 #10
0
 if ($count % 100 == 0) {
     echo '.';
 }
 // Key fields data
 $key_values = array();
 foreach ($key_fields as $key_field) {
     $temp_key_v = array_get_index($record, $key_field, NULL);
     if (is_null($temp_key_v)) {
         // Primary key field must be there
         continue 2;
     }
     $key_values[$key_field] = $temp_key_v;
 }
 //end foreach
 // Value field to check
 $org_value = array_get_index($record, $value_field, '');
 if (empty($org_value)) {
     continue;
 }
 // Replace the "./?a=xxx" links in the content
 $new_value = _rewrite_assetid_links($org_value, $record, $table == 'ast_mdata_val');
 if ($new_value !== FALSE && !$report_mode) {
     // Generate update sql
     $bind_vars = array();
     $set_sql = $value_field . '=:' . $value_field . '_v';
     $bind_vars[$value_field . '_v'] = $new_value;
     $where_sql = array();
     foreach ($key_values as $field_name => $value) {
         $where_sql[] = $field_name . '=:' . $field_name . '_k';
         $bind_vars[$field_name . '_k'] = $value;
     }
コード例 #11
0
                     }
                     // Cut it down to a certain number of characters anyway
                     $value = ellipsisize($value, 120);
                     $value = preg_replace('/(' . addslashes(implode('|', $words)) . ')/i', '<span class="sq-backend-search-results-highlight">$1</span>', $value);
                     // remove \r and replace \n with line breaks
                     $this_detail[] = $name_detail . '<br/><em>' . str_replace("\r", '', str_replace("\n", '<br/>', $value)) . '</em>';
                 }
                 //end foreach
             }
             //end foreach
             $asset_name = $GLOBALS['SQ_SYSTEM']->am->getAssetInfo(array($result_assetid), 'image', FALSE, 'name');
             $asset_title = $GLOBALS['SQ_SYSTEM']->am->getAttributeValuesByName('title', 'image', array($result_assetid));
             $asset_alt = $GLOBALS['SQ_SYSTEM']->am->getAttributeValuesByName('alt', 'image', array($result_assetid));
             $asset_width = $GLOBALS['SQ_SYSTEM']->am->getAttributeValuesByName('width', 'image', array($result_assetid));
             $asset_height = $GLOBALS['SQ_SYSTEM']->am->getAttributeValuesByName('height', 'image', array($result_assetid));
             $result_list[] = array('tag_line' => get_asset_tag_line($result_assetid, 'javascript:set_asset_finder_from_search(\'' . $result_assetid . '\', \'' . htmlspecialchars(array_get_index($asset_title, $result_assetid, ''), ENT_QUOTES) . '\', \'\', \'0\', \'' . array_get_index($asset_name, $result_assetid, '') . '\', \'' . addslashes(array_get_index($asset_alt, $result_assetid, '')) . '\', \'' . array_get_index($asset_width, $result_assetid, '') . '\', \'' . array_get_index($asset_height, $result_assetid, '') . '\');'), 'detail' => implode($this_detail, '<br/>'));
         }
         //end foreach
     }
     //end if
 }
 //end else
 // Are there any results? If not, put in a "search failed" box, otherwise
 // build the results box
 if (empty($results) && empty($found_asset_line)) {
     $box_title = translate('Search Failed');
     $html = sprintf(translate('Search for "%s" found nothing. Try Again'), addslashes($search_for));
     $style_base = 'search-failed';
 } else {
     $box_title = translate('Search Results');
     if (!empty($found_asset_line)) {
コード例 #12
0
        echo 'Resuming HIPO Job ';
        echo ucwords(str_replace('_', ' ', $source_job_type)) . "\t";
        // Do itttttt
        $status = $source_job->process();
        if ($status) {
            echo '[  OK  ]';
        } else {
            echo '[  !!  ]';
        }
        //end if
        echo "\n";
        break;
    case 'k':
        // Kill kill kill
        $source_code_name = array_get_index($source_jobs[$actual_choice], 'source_code_name', '');
        $source_job_type = array_get_index($source_jobs[$actual_choice], 'job_type', '');
        if (empty($source_code_name)) {
            exit;
        }
        $source_job = $hh->getJob($source_code_name);
        if (is_null($source_job)) {
            exit;
        }
        echo 'Aborting HIPO Job ';
        echo ucwords(str_replace('_', ' ', $source_job_type)) . "\n";
        $source_job->abort();
        break;
    case 'q':
    default:
        exit;
}
コード例 #13
0
* @author  Benjamin Pearson <*****@*****.**>
* @version $Revision: 1.4 $
* @package MySource_Matrix
*/
require_once dirname(__FILE__) . '/../../../../core/include/init.inc';
require_once SQ_LIB_PATH . '/html_form/html_form.inc';
require_once SQ_LIB_PATH . '/backend_search/backend_search.inc';
if (empty($GLOBALS['SQ_SYSTEM']->user) || !($GLOBALS['SQ_SYSTEM']->user->canAccessBackend() || $GLOBALS['SQ_SYSTEM']->user->type() == 'simple_edit_user' || method_exists($GLOBALS['SQ_SYSTEM']->user, 'isShadowSimpleEditUser') && $GLOBALS['SQ_SYSTEM']->user->isShadowSimpleEditUser())) {
    exit;
}
if (Backend_Search::isAvailable()) {
    $quick_search_for_text = translate('Enter keywords, asset ID or URL');
} else {
    $quick_search_for_text = translate('Enter asset ID or URL');
}
$search_for = trim(array_get_index($_GET, 'quick-search-for', ''));
// If we are searching for something
if ($search_for != '') {
    // check for a url first
    $asset_by_url = $GLOBALS['SQ_SYSTEM']->am->getAssetFromURL('', strip_url($search_for, TRUE), TRUE, TRUE);
    if (!empty($asset_by_url) && ($asset_by_url->type() != 'file' || !$asset_by_url->readAccess())) {
        $asset_by_url = NULL;
    }
    if (assert_valid_assetid($search_for, '', TRUE, FALSE)) {
        $asset_by_id = $GLOBALS['SQ_SYSTEM']->am->getAsset($search_for, '', TRUE);
        if (!empty($asset_by_id) && ($asset_by_id->type() != 'file' || !$asset_by_id->readAccess())) {
            $asset_by_id = NULL;
        }
    }
    // Only search for files (strict type check)
    $results = Backend_Search::processSearch($search_for, array(), array('file' => FALSE));
コード例 #14
0
ファイル: CountDistinct.h.php プロジェクト: gokulp/grokit
/**
 *  A GLA that counts the number of distinct elements by keeping track of the
 *  distinct elements.
 *
 *  Unless an exact count of the distinct is absolutely needed, consider using
 *  an approximation of the distinct, such as a Bloom Filter.
 */
function CountDistinct(array $t_args, array $input, array $output)
{
    grokit_assert(\count($output) == 1, 'CountDistinct should have only 1 output, ' . \count($output) . 'given');
    $outputName = array_keys($output)[0];
    $outputType = array_get_index($output, 0);
    if (is_null($outputType)) {
        $outputType = lookupType('BASE::BIGINT');
    }
    $output[$outputName] = $outputType;
    grokit_assert($outputType->is('numeric'), 'CountDistinct output must be numeric!');
    $useMCT = get_default($t_args, 'use.mct', true);
    $keepHashes = get_default($t_args, 'mct.keep.hashes', false);
    $initSize = get_default($t_args, 'init.size', 65536);
    $nullCheck = get_default($t_args, 'null.check', false);
    grokit_assert(is_bool($useMCT), 'CountDistinct use.mct argument must be boolean');
    grokit_assert(is_integer($initSize), 'Distinct init.size argument must be an integer');
    grokit_assert($initSize > 0, 'Distinct init.size argument must be positive');
    grokit_assert(is_bool($keepHashes), 'CountDistinct mct.keep.hashes argument must be boolean');
    $distTmpArgs = ['use.mct' => $useMCT, 'init.size' => $initSize, 'mct.keep.hashes' => $keepHashes, 'null.check' => $nullCheck];
    $gla = lookupGLA('BASE::DISTINCT', $distTmpArgs, $input, $input);
    $className = generate_name('CountDistinct');
    ?>
class <?php 
    echo $className;
    ?>
 {
    using Distinct = <?php 
    echo $gla->value();
    ?>
;

    Distinct distinctGLA;

public:

    <?php 
    echo $className;
    ?>
(void):
        distinctGLA()
    { }

    ~<?php 
    echo $className;
    ?>
(void) { }

    void AddItem(<?php 
    echo const_typed_ref_args($input);
    ?>
) {
        distinctGLA.AddItem(<?php 
    echo args($input);
    ?>
);
    }

    void AddState(<?php 
    echo $className;
    ?>
 & o) {
        distinctGLA.AddState(o.distinctGLA);
    }

    void GetResult(<?php 
    echo $outputType;
    ?>
 & <?php 
    echo $outputName;
    ?>
) {
        <?php 
    echo $outputName;
    ?>
 = distinctGLA.get_countDistinct();
    }
};
<?php 
    return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single'];
}
コード例 #15
0
 MatrixDAL::executeSql('ALTER TABLE sq_suite_product DROP COLUMN knows_me_as');
 // Populate data into not null columns.
 MatrixDAL::executeSql("UPDATE sq_suite_product SET suiteid=sq_suite_seq.nextVal");
 MatrixDAL::executeSql("UPDATE sq_suite_product SET url=''");
 $products = MatrixDAL::executeSqlAssoc('SELECT suiteid, connection FROM sq_suite_product');
 foreach ($products as $product) {
     $suiteid = array_get_index($product, 'suiteid', NULL);
     $connection = array_get_index($product, 'connection', NULL);
     if ($suiteid === NULL || $connection === NULL) {
         continue;
     }
     $connection = @unserialize($connection);
     if ($connection === FALSE) {
         continue;
     }
     $url = array_get_index($connection, 'url', NULL);
     if ($url === NULL) {
         continue;
     }
     unset($connection['url']);
     $query = MatrixDAL::preparePdoQuery('UPDATE sq_suite_product SET url=:url, connection=:connection WHERE suiteid=:id');
     MatrixDAL::bindValueToPdo($query, 'url', $url);
     MatrixDAL::bindValueToPdo($query, 'connection', serialize($connection));
     MatrixDAL::bindValueToPdo($query, 'id', $suiteid);
     MatrixDAL::execPdoQuery($query);
 }
 //end foreach
 // Set the not null constraint on columns.
 MatrixDAL::executeSql('ALTER TABLE sq_suite_product MODIFY suiteid NOT NULL');
 MatrixDAL::executeSql('ALTER TABLE sq_suite_product MODIFY url NOT NULL');
 // Set the new constraints and keys
コード例 #16
0
ファイル: Max.h.php プロジェクト: gokulp/grokit
function Max(array $t_args, array $input, array $output)
{
    grokit_assert(\count($output) >= 1, 'Max GLA  produces at least one output!');
    grokit_assert(\count($output) == \count($input), 'Max GLA should have the same number of inputs and outputs');
    $nValues = \count($output);
    $inputNames = array_keys($input);
    $outputNames = array_keys($output);
    // Outputs should be the same type as the inputs
    for ($index = 0; $index < $nValues; $index++) {
        array_set_index($output, $index, array_get_index($input, $index));
    }
    $name = generate_name('Max_');
    ?>
class <?php 
    echo $name;
    ?>
 {
    uintmax_t count;

<?php 
    foreach ($output as $k => $v) {
        ?>
    <?php 
        echo $v;
        ?>
 _<?php 
        echo $k;
        ?>
;
<?php 
    }
    // foreach output
    ?>

public:
    <?php 
    echo $name;
    ?>
() :
<?php 
    foreach ($output as $k => $v) {
        ?>
        _<?php 
        echo $k;
        ?>
(),
<?php 
    }
    // foreach output
    ?>
        count(0)
    { }

    void AddItem( <?php 
    echo const_typed_ref_args($input);
    ?>
 ) {
        if( count > 0 ) {
<?php 
    for ($index = 0; $index < $nValues; $index++) {
        ?>
            _<?php 
        echo $outputNames[$index];
        ?>
 = std::max(_<?php 
        echo $outputNames[$index];
        ?>
, <?php 
        echo $inputNames[$index];
        ?>
);
<?php 
    }
    // foreach value
    ?>
        } else {
<?php 
    for ($index = 0; $index < $nValues; $index++) {
        ?>
            _<?php 
        echo $outputNames[$index];
        ?>
 = <?php 
        echo $inputNames[$index];
        ?>
;            
<?php 
    }
    // foreach value
    ?>
        }

        count++;
    }
    void AddState( <?php 
    echo $name;
    ?>
 & o ) {
        if (count > 0 && o.count > 0) {
<?php 
    for ($index = 0; $index < $nValues; $index++) {
        ?>
            _<?php 
        echo $outputNames[$index];
        ?>
 = std::max(_<?php 
        echo $outputNames[$index];
        ?>
, o._<?php 
        echo $outputNames[$index];
        ?>
);
<?php 
    }
    // foreach value
    ?>
        } else if(o.count > 0) { // count == 0
<?php 
    for ($index = 0; $index < $nValues; $index++) {
        ?>
            _<?php 
        echo $outputNames[$index];
        ?>
 = o._<?php 
        echo $outputNames[$index];
        ?>
;            
<?php 
    }
    // foreach value
    ?>
        }
        // Otherwise, count > 0 && o.count == 0, so just keep our values

        count += o.count;
    }

    void GetResult(<?php 
    echo typed_ref_args($output);
    ?>
) {
<?php 
    foreach ($output as $k => $v) {
        ?>
        <?php 
        echo $k;
        ?>
 = _<?php 
        echo $k;
        ?>
;
<?php 
    }
    // foreach output
    ?>
    }
};
<?php 
    return ['kind' => 'GLA', 'name' => $name, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'system_headers' => ['algorithm', 'cstdint']];
}
コード例 #17
0
 /**
  * Normal stuff we need to get ready before we do anything else
  *
  * @return boolean
  * @access public
  */
 function load()
 {
     $db =& $GLOBALS['SQ_SYSTEM']->db;
     $GLOBALS['SQ_SYSTEM']->changeDatabaseConnection('db');
     $dbtype = $this->_getDbType();
     // load our tables from the database
     switch ($dbtype) {
         case 'oci':
             $sql = "SELECT object_name AS table_name FROM user_objects WHERE object_type = 'TABLE'";
             break;
         default:
             $sql = "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE'";
             break;
     }
     //end switch
     $results = MatrixDAL::executeSqlAssoc($sql);
     $tables = array();
     foreach ($results as $table) {
         $table_name = array_get_index($table, 'table_name', '');
         if (!empty($table_name)) {
             $tables[] = strtolower($table_name);
         }
         //end if
     }
     //end foreach
     $this->tables = $tables;
     if (empty($this->tables)) {
         return FALSE;
     }
     return TRUE;
 }
コード例 #18
0
ファイル: Segmented.h.php プロジェクト: gokulp/grokit
function Segmenter(array $t_args, array $input, array $output, array $given_states)
{
    $resType = ['fragment', 'multi'];
    $system_headers = ['array', 'vector', 'memory', 'cinttypes', 'unordered_map'];
    $user_headers = ['HashFunctions.h'];
    $lib_headers = [];
    $preferFragment = get_default($t_args, 'inner.prefer.fragment', false);
    $wantedRes = $preferFragment ? ['fragment', 'multi'] : ['multi', 'fragment'];
    $nInputs = \count($input);
    grokit_assert($nInputs > 1, 'Segmenter: Not enough inputs specified!');
    $keyName = array_keys($input)[0];
    $keyType = array_get_index($input, 0);
    $innerInputs = array_slice($input, 1, $nInputs - 1, true);
    $gla = get_first_key($t_args, ['gla', 'GLA', 0]);
    grokit_assert(is_gla($gla), 'Segmenter: [gla] argument must be a valid GLA');
    $gla = $gla->apply($innerInputs, $output, $given_states);
    $n_passes = get_default($t_args, 'passes', 1);
    grokit_assert(is_int($n_passes), 'Segmenter: [passes] argument must be an integer');
    grokit_assert($n_passes > 0, 'Segmenter: [passes] argument must be > 0');
    $libraries = $gla->libraries();
    $innerRes = get_first_value($gla->result_type(), $wantedRes);
    $innerInputs = $gla->input();
    $innerOutput = $gla->output();
    $input = array_merge([$keyName => $keyType], $innerInputs);
    $output = $innerOutput;
    $segments = get_default($t_args, 'segments', 64);
    $constState = lookupResource('BASE::SegmenterState', ['gla' => $gla, 'passes' => $n_passes, 'segments' => $segments]);
    $className = generate_name('Segmenter_');
    $savedArgs = [];
    $cArgs = [];
    $innerCArgs = [];
    if ($gla->configurable()) {
        $savedArgs['json_init'] = 'Json::Value';
        $cArgs['json_init'] = 'Json::Value';
        $innerCArgs[] = 'json_init';
    }
    $cArgs['const_state'] = $constState;
    if ($gla->has_state()) {
        $innerCArgs[] = 'constState.inner_cstate';
    }
    $cstStr = \count($innerCArgs) > 0 ? '(' . implode(',', $innerCArgs) . ')' : '';
    grokit_assert(!$gla->iterable(), 'Segementer does not support iterable GLAs');
    $iterable = $n_passes > 1;
    ?>

class <?php 
    echo $className;
    ?>
 {
private:

    using ConstantState = <?php 
    echo $constState;
    ?>
;
    using SplitState = ConstantState::SplitState;

    static constexpr const size_t NUM_STATES = SplitState::NUM_STATES;

    using InnerGLA = <?php 
    echo $gla;
    ?>
;
    using InnerGLAPtr = std::unique_ptr<InnerGLA>;
    using GLA_Array = std::array<InnerGLAPtr, NUM_STATES>;

public:
    using size_type = std::size_t;

<?php 
    if ($innerRes == 'fragment') {
        ?>
    class Iterator {
    private:
        InnerGLA * gla;
        int fragmentNum;

        InnerGLA::Iterator * innerIter;

    public:
        Iterator( InnerGLA * _gla, int _fragmentNum, int _innerFrag ) :
            gla(_gla), fragmentNum(_fragmentNum),
            innerIter(nullptr)
        {
            innerIter = gla->Finalize(_innerFrag);
        }

        ~Iterator(void) {
            if( innerIter != nullptr ) {
                delete innerIter;
                innerIter = nullptr;
            }
        }

        bool GetNextResult( <?php 
        echo typed_ref_args($gla->output());
        ?>
 ) {
            return innerIter->GetNextResult(<?php 
        echo args($gla->output());
        ?>
);
        }

        int FragmentNumber() {
            return fragmentNum;
        }
    };
<?php 
    } else {
        // if inner result type is fragment
        ?>
    class Iterator {
    private:
        InnerGLA * gla;
        int fragmentNum;

    public:
        Iterator( InnerGLA * _gla, int fragNo ) : gla(_gla), fragmentNum(fragNo) {
            gla->Finalize();
        }

        ~Iterator(void) { }

        bool GetNextResult( <?php 
        echo typed_ref_args($gla->output());
        ?>
 ) {
            return gla->GetNextResult(<?php 
        echo args($gla->output());
        ?>
);
        }

        int FragmentNumber() {
            return fragmentNum;
        }
    };
<?php 
    }
    // if inner result type is multi
    ?>

private:

    const ConstantState & constState;
    GLA_Array localState;

    // Iteration state for multi result type
    int numFrags;
    int multiFragNo;
    Iterator * multiIter;

<?php 
    if ($innerRes == 'fragment') {
        ?>
    using frag_info = std::pair<int, int>;
    using frag_map_t = std::unordered_map<int, frag_info>;
    frag_map_t fragMap;
<?php 
    }
    ?>


<?php 
    foreach ($savedArgs as $name => $type) {
        ?>
    const <?php 
        echo $type;
        ?>
 <?php 
        echo $name;
        ?>
;
<?php 
    }
    // foreach saved arg
    ?>

public:

    // Constructor
    <?php 
    echo $className;
    ?>
( <?php 
    echo const_typed_ref_args($cArgs);
    ?>
 ) :
        constState(const_state)
        , localState()
        , numFrags(0)
        , multiFragNo(0)
        , multiIter(nullptr)
<?php 
    if ($innerRes == 'fragment') {
        ?>
        , fragMap()
<?php 
    }
    foreach ($savedArgs as $name => $type) {
        ?>
        , <?php 
        echo $name;
        ?>
(<?php 
        echo $name;
        ?>
)
<?php 
    }
    // foreach constructor arg to save
    ?>
    {
        for( auto & elem : localState ) {
            elem.reset(new InnerGLA<?php 
    echo $cstStr;
    ?>
);
        }
    }

    void AddItem( <?php 
    echo const_typed_ref_args($input);
    ?>
 ) {
        uint64_t hashVal = CongruentHash(Hash(<?php 
    echo $keyName;
    ?>
), H_b + 1);
        uint64_t passNum = (hashVal / NUM_STATES) % ConstantState::N_PASSES;
        uint64_t segNum = hashVal % NUM_STATES;

<?php 
    if ($n_passes > 1) {
        ?>
        if( passNum != constState.pass ) {
            return;
        }
<?php 
    }
    // more than 1 pass
    ?>
        localState[segNum]->AddItem(<?php 
    echo args($innerInputs);
    ?>
);
    }

    void ChunkBoundary(void) {
        // Merge local states into the global state

        SplitState & globalStates = constState.segments;

        int theseAreOk[NUM_STATES];
        for( int i = 0; NUM_STATES > i; i++ ) {
            theseAreOk[i] = 1;
        }

        int segsLeft = NUM_STATES;

        while( segsLeft > 0 ) {
            InnerGLA * checkedOut = nullptr;
            int whichOne = globalStates.CheckOutOne( theseAreOk, checkedOut );

            if( checkedOut == NULL ) {
                checkedOut = new InnerGLA<?php 
    echo $cstStr;
    ?>
;
            }

            checkedOut->AddState( *(localState[whichOne]) );

            globalStates.CheckIn( whichOne, checkedOut );

            theseAreOk[whichOne] = 0;
            segsLeft--;
        }

        // Re-initialize the local states
        for( auto & elem : localState ) {
<?php 
    if ($gla->is('resettable')) {
        ?>
            elem->Reset();
<?php 
    } else {
        // if resettable
        ?>
            elem.reset(new InnerGLA<?php 
        echo $cstStr;
        ?>
);
<?php 
    }
    // if not resettable
    ?>
        }
    }

    void AddState( <?php 
    echo $className;
    ?>
 & o ) {
        // Do nothing
    }

    void Finalize() {
        SplitState & globalStates = constState.segments;

        if( multiIter != nullptr)
            delete multiIter;

        multiFragNo = 0;
<?php 
    if ($innerRes == 'fragment') {
        ?>
        frag_info fInfo = fragMap[multiFragNo];
        multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo,
            fInfo.second);
<?php 
    } else {
        ?>
        multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo);
<?php 
    }
    ?>
    }

    bool GetNextResult(<?php 
    echo typed_ref_args($output);
    ?>
) {
        bool gotResult = false;
        SplitState & globalStates = constState.segments;

        while( (multiFragNo < numFrags && multiIter != nullptr) && !gotResult ) {
            gotResult = multiIter->GetNextResult(<?php 
    echo args($output);
    ?>
);

            if( !gotResult ) {
                multiFragNo++;
                delete multiIter;

                if( numFrags > multiFragNo ) {
<?php 
    if ($innerRes == 'fragment') {
        ?>
                    frag_info fInfo = fragMap[multiFragNo];
                    multiIter = new Iterator(globalStates.Peek(fInfo.first), multiFragNo,
                        fInfo.second);
<?php 
    } else {
        ?>
                    multiIter = new Iterator(globalStates.Peek(multiFragNo), multiFragNo);
<?php 
    }
    ?>
                } else {
                    multiIter = nullptr;
                }
            }
        }

        return gotResult;
    }

    int GetNumFragments(void) {
<?php 
    if ($innerRes == 'fragment') {
        ?>
        SplitState & globalStates = constState.segments;
        numFrags = 0;

        for (int i = 0; i < NUM_STATES; i++) {
            int curFrags = globalStates.Peek(i)->GetNumFragments();

            for (int curFrag = 0; curFrag < curFrags; curFrag++) {
                fragMap[numFrags] = frag_info(i, curFrag);

                numFrags++;
            }
        }
<?php 
    } else {
        ?>
        numFrags = NUM_STATES;
<?php 
    }
    ?>
        return numFrags;
    }

    Iterator * Finalize( int fragment ) {
        SplitState & globalStates = constState.segments;

<?php 
    if ($innerRes == 'fragment') {
        ?>
        frag_info info = fragMap[fragment];
        return new Iterator(globalStates.Peek(info.first), fragment, info.second);
<?php 
    } else {
        ?>
        return new Iterator(globalStates.Peek(fragment), fragment);
<?php 
    }
    ?>
    }

    bool GetNextResult( Iterator * it, <?php 
    echo typed_ref_args($output);
    ?>
 ) {
        bool ret = it->GetNextResult(<?php 
    echo args($output);
    ?>
);

        return ret;
    }

<?php 
    if ($iterable) {
        ?>
    bool ShouldIterate( ConstantState & modible ) {
        modible.pass++;

        return modible.pass < ConstantState::N_PASSES;
    }

    void PostFinalize() {
        constState.segments.Reset();
    }
<?php 
    }
    // iterable
    ?>

<?php 
    if ($gla->is('finite container')) {
        ?>
    size_type size() {
        SplitState & globalStates = constState.segments;
        size_type s = 0;
        for( int i = 0; NUM_STATES > i; i++ ) {
            InnerGLA * ptr = globalStates.Peek(i);
            s += ptr->size();
        }

        return s;
    }

    size_type size(int frag) {
        SplitState & globalStates = constState.segments;
        return globalStates.Peek(frag)->size();
    }
<?php 
    }
    // if the gla is a container
    ?>
};

typedef <?php 
    echo $className;
    ?>
::Iterator <?php 
    echo $className;
    ?>
_Iterator;

<?php 
    return ['kind' => 'GLA', 'name' => $className, 'system_headers' => $system_headers, 'user_headers' => $user_headers, 'lib_headers' => $lib_headers, 'libraries' => $libraries, 'input' => $input, 'output' => $output, 'result_type' => $resType, 'generated_state' => $constState, 'required_states' => $gla->req_states(), 'chunk_boundary' => true, 'configurable' => $gla->configurable(), 'iterable' => $iterable, 'post_finalize' => $iterable, 'intermediates' => true];
}
コード例 #19
0
ファイル: BloomFilter.h.php プロジェクト: gokulp/grokit
/**
 *  A GLA that estimates the cardinality of a dataset using a bloom filter of
 *  a configurable size.
 *
 *  Note: This filter has very high performance, so long as all of the states
 *  fit into cache, preferably L1 or L2, but L3 is also fine. Once the states
 *  are large enough that all of them cannot fit inside L3 cache at the same
 *  time, performance takes a nose dive (4x loss minimum).
 */
function BloomFilter(array $t_args, array $input, array $output)
{
    grokit_assert(\count($output) == 1, 'BloomFilter produces only 1 value, ' . \count($output) . ' outputs given.');
    $outputName = array_keys($output)[0];
    $outputType = array_get_index($output, 0);
    if (is_null($outputType)) {
        $outputType = lookupType('BASE::BIGINT');
    }
    $output[$outputName] = $outputType;
    grokit_assert($outputType->is('numeric'), 'BloomFilter output must be numeric!');
    $exp = get_first_key_default($t_args, ['exponent'], 16);
    grokit_assert(is_integer($exp), 'BloomFilter exponent must be an integer.');
    grokit_assert($exp > 0 && $exp < 64, 'BloomFilter exponent must be in range (0,64), ' . $exp . ' given.');
    $nullCheck = get_default($t_args, 'null.check', false);
    $nullable = [];
    if (is_bool($nullCheck)) {
        foreach ($input as $name => $type) {
            $nullable[$name] = $nullCheck;
        }
    } else {
        if (is_array($nullCheck)) {
            foreach ($input as $name => $type) {
                $nullable[$name] = false;
            }
            foreach ($nullCheck as $index => $n) {
                grokit_assert(is_string($n), 'BloomFilster null.check has invalid value at position ' . $index);
                grokit_assert(array_key_exists($n, $nullable), 'BloomFilster null.check has unknown input ' . $n . ' at position ' . $index);
                $nullable[$n] = true;
            }
        } else {
            grokit_error('BloomFilster null.check must be boolean or list of inputs to check for nulls');
        }
    }
    $debug = get_default($t_args, 'debug', 0);
    $bits = pow(2, $exp);
    $bytes = ceil($bits / 8.0);
    // Calculate the number of bits set for every possible value of a byte
    $nBits = [];
    for ($i = 0; $i < 256; $i++) {
        $n = $i;
        $b = 0;
        while ($n > 0) {
            $n &= $n - 1;
            $b++;
        }
        $nBits[$i] = $b;
    }
    $className = generate_name('BloomFilter');
    ?>
class <?php 
    echo $className;
    ?>
 {
    static constexpr size_t BITS = <?php 
    echo $bits;
    ?>
;
    static constexpr size_t BYTES = <?php 
    echo $bytes;
    ?>
;
    static constexpr size_t MASK = BITS - 1;
    static constexpr std::array<unsigned char, 256> BITS_SET = { <?php 
    echo implode(', ', $nBits);
    ?>
 };
    static constexpr std::array<unsigned char, 8> BIT_MASKS = {
        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80
    };

    size_t count;

    std::array<unsigned char, BYTES> set;
    //unsigned char set[BYTES];
    //std::bitset<BITS> set;

public:
    <?php 
    echo $className;
    ?>
() : count(0), set() {
        for( size_t i = 0; i < BYTES; i++ ) { //>
            set[i] = 0;
        }
    }

    ~<?php 
    echo $className;
    ?>
() { }

    void AddItem( <?php 
    echo const_typed_ref_args($input);
    ?>
 ) {
        count++;
<?php 
    foreach ($nullable as $name => $check) {
        if ($check) {
            ?>
        if( IsNull( <?php 
            echo $name;
            ?>
 ) ) return;
<?php 
        }
        // if checking for nulls
    }
    // foreach input
    ?>
        size_t hashVal = H_b;
<?php 
    foreach ($input as $name => $type) {
        ?>
        hashVal = CongruentHash(Hash(<?php 
        echo $name;
        ?>
), hashVal);
<?php 
    }
    // foreach input
    ?>
        hashVal = hashVal & MASK;
        const size_t bucket = hashVal >> 3;
        const size_t bucket_index = hashVal & 0x07;
        const unsigned char mask = BIT_MASKS[bucket_index];
        set[bucket] |= mask;
    }

    void AddState( <?php 
    echo $className;
    ?>
 & o ) {
        count += o.count;
        for( size_t i = 0; i < BYTES; i++ ) { //>
            set[i] |= o.set[i];
        }
    }

    void GetResult( <?php 
    echo $outputType;
    ?>
 & <?php 
    echo $outputName;
    ?>
 ) {
        size_t nBitsSet = 0;
        constexpr long double bits = static_cast<long double>(BITS);
        for( size_t i = 0; i < BYTES; i++ ) { //>
            nBitsSet += BITS_SET[set[i]];
        }
        long double bitsSet = static_cast<long double>(nBitsSet);

        if( nBitsSet == BITS ) {
            // All Bits set, just give the cardinality as an estimate.
            <?php 
    echo $outputName;
    ?>
 = count;
        } else {
            long double cardinality = - bits * std::log(1 - (bitsSet / bits));
            <?php 
    echo $outputName;
    ?>
 = cardinality;
        }

<?php 
    if ($debug > 0) {
        ?>
        std::cout << "BloomFilter:"
            << " bitsSet(" << bitsSet << ")"
            << " bits(" << bits << ")"
            << " cardinality(" << cardinality << ")"
            << " output(" << <?php 
        echo $outputName;
        ?>
 << ")"
            << std::endl;; //>
<?php 
    }
    // if debugging enabled
    ?>
    }
};

// Storage for static members
constexpr std::array<unsigned char, 256> <?php 
    echo $className;
    ?>
::BITS_SET;
constexpr std::array<unsigned char, 8> <?php 
    echo $className;
    ?>
::BIT_MASKS;
<?php 
    $system_headers = ['cmath', 'array'];
    if ($debug > 0) {
        $system_headers[] = 'iostream';
    }
    return ['kind' => 'GLA', 'name' => $className, 'input' => $input, 'output' => $output, 'result_type' => 'single', 'user_headers' => ['HashFunctions.h'], 'system_headers' => $system_headers];
}
コード例 #20
0
 * report invalid new url remaps
 */
if (getCLIArg('invalid_new_url')) {
    echo "\n## Invalid New URL ##\n";
    $sql = 'SELECT * FROM sq_ast_lookup_remap WHERE expires > :time OR expires is null';
    $query = MatrixDAL::preparePdoQuery($sql);
    $time = ts_iso8601(time());
    MatrixDAL::bindValueToPdo($query, 'time', $time);
    $result = MatrixDAL::executePdoAll($query);
    $affected_result = array();
    $root_urls = explode("\n", SQ_CONF_SYSTEM_ROOT_URLS);
    foreach ($result as $index => $url_info) {
        $new_url_parts = parse_url($url_info['remap_url']);
        $protocol = array_get_index($new_url_parts, 'scheme', NULL);
        if (!empty($protocol)) {
            $url = array_get_index($new_url_parts, 'host', '') . array_get_index($new_url_parts, 'path', '');
            //  does it match with root matrix url
            $is_matrix_url = FALSE;
            foreach ($root_urls as $root_url) {
                if (strpos($url, $root_url . '/') === 0 || $url === $root_url) {
                    $is_matrix_url = TRUE;
                    break;
                }
            }
            // if it's not a Matrix URL, no need to show warning
            if ($is_matrix_url) {
                // if it's one of special url, no need to show warning
                if (strpos($url, '__data') === FALSE && strpos($url, '__lib') === FALSE && strpos($url, '__fudge') === FALSE) {
                    $url_asset = $GLOBALS['SQ_SYSTEM']->am->getAssetFromURL($protocol, $url, TRUE, TRUE);
                    $count_query = 'SELECT count(*) FROM sq_ast_lookup_remap WHERE url = :url ';
                    $count_result = MatrixDAL::preparePdoQuery($count_query);
コード例 #21
0
}
if (!is_dir($SYSTEM_ROOT) || !is_readable($SYSTEM_ROOT . '/core/include/init.inc')) {
    echo "ERROR: Path provided doesn't point to a Matrix installation's System Root. Please provide correct path and try again.\n";
    echo 'Usage: ' . basename(__FILE__) . " SYSTEM_ROOT [NEW_PASSWORD]\n\n";
    echo "       If NEW_PASSWORD is not provided, it will be reset to 'root'\n";
    exit;
}
require_once $SYSTEM_ROOT . '/core/include/init.inc';
// Get the root user
$root_user =& $GLOBALS['SQ_SYSTEM']->am->getSystemAsset('root_user');
// log in as root :P
if (!$GLOBALS['SQ_SYSTEM']->setCurrentUser($root_user)) {
    echo "ERROR: Failed login in as root user\n";
    exit;
}
// try to lock the root user
if (!$GLOBALS['SQ_SYSTEM']->am->acquireLock($root_user->id, 'attributes')) {
    trigger_error("Couldn't get lock\n", E_USER_ERROR);
}
$current_run_level = $GLOBALS['SQ_SYSTEM']->getRunLevel();
$GLOBALS['SQ_SYSTEM']->setRunLevel($current_run_level - SQ_SECURITY_PASSWORD_VALIDATION);
$password = array_get_index($argv, 2, 'root');
if (!$root_user->setAttrValue('password', $password)) {
    trigger_error("Couldn't set password\n", E_USER_ERROR);
}
if (!$root_user->saveAttributes()) {
    trigger_error("Couldn't save attributes \n", E_USER_ERROR);
}
$GLOBALS['SQ_SYSTEM']->restoreRunLevel();
$GLOBALS['SQ_SYSTEM']->am->releaseLock($root_user->id, 'attributes');
echo 'Root User Password now reset to "' . $password . '", please login and change.', "\n";
コード例 #22
-2
/**
 * Fixes the char encoding in the given tables in the database
 *
 * @param int		$root_node		Assetid of rootnode, all childern of rootnode will be processed for char replacement
 * @param array		$tables			DB tables and colunms info
 * @param boolean	$rollback		If TRUE process rollback tables, else process regular tables
 *
 * @return void
 */
function fix_db($root_node, $tables, $rollback)
{
    global $reportOnly;
    $tables_info = get_tables_info();
    // All the Matrix attribute types with serialised value
    $serialsed_attrs = array('option_list', 'email_format', 'parameter_map', 'serialise', 'http_request', 'oauth');
    // Get the list of attrids of the type 'serialise'
    $sql = "SELECT attrid FROM sq_ast_attr WHERE type IN ('" . implode("','", $serialsed_attrs) . "')";
    $serialise_attrids = array_keys(MatrixDAL::executeSqlGrouped($sql));
    if ($root_node != 1) {
        // Get the targetted asset list
        $target_assetids = array_keys($GLOBALS['SQ_SYSTEM']->am->getChildren($root_node));
        // Since we include the root node, target assetids will always contain atleast one asset id
        array_unshift($target_assetids, $root_node);
        echo "\n\nNumber of assets to look into : " . count($target_assetids) . " \n";
        // Go through 50 assets at a time. Applicable to asset specific tables only
        $chunks = array_chunk($target_assetids, 50);
        $chunks_count = count($chunks);
    }
    $errors_count = 0;
    $warnings_count = 0;
    $records_fixed_count = 0;
    $invalid_asset_records = array();
    // Assets that will require filesystem content regeneration
    $affected_assetids = array();
    $GLOBALS['SQ_SYSTEM']->changeDatabaseConnection('db2');
    // Counter to count the number of records accessed/processed
    $count = 0;
    foreach ($tables as $table_data) {
        $table_records_count = 0;
        $table = isset($table_data['table']) ? $table_data['table'] : '';
        if (empty($table)) {
            continue;
        }
        $key_fields = isset($tables_info[$table]['primary_key']) ? $tables_info[$table]['primary_key'] : '';
        if (empty($key_fields)) {
            echo "\n" . 'Ignoring table "' . $table . '". Table info for this table not found' . " \n";
            continue;
        }
        $value_fields = isset($table_data['values']) ? $table_data['values'] : '';
        if (empty($value_fields)) {
            // Nothing to check
            continue;
        }
        if ($rollback) {
            // Make sure table has rollback trigggers enabled, otherwise it will have rollback table
            if (isset($tables_info[$table]['rollback']) && $tables_info[$table]['rollback']) {
                // Add rollback table primary key field to the table's keys
                $key_fields[] = 'sq_eff_from';
            } else {
                // This table does not has corresponding rollback table
                continue;
            }
        }
        // Prepend table prefix
        $table = !$rollback ? 'sq_' . $table : 'sq_rb_' . $table;
        $asste_specific_table = $table_data['asset_assoc'];
        $select_fields = array_merge($value_fields, $key_fields);
        if ($asste_specific_table && !in_array('assetid', $select_fields)) {
            $select_fields[] = 'assetid';
        }
        if ($root_node == 1) {
            if ($asste_specific_table) {
                // When running system wide, get the asset list from the respective db table
                $sql = "SELECT DISTINCT assetid FROM " . $table;
                $target_assetids = array_keys(MatrixDAL::executeSqlGrouped($sql));
                // Go through 50 assets at a time. Applicable to asset specific tables only
                $chunks = array_chunk($target_assetids, 50);
            } else {
                // Dummy assetids chuck just so that we can get into next loop
                $chunks = array(array());
            }
        }
        echo "\nChecking " . $table . " .";
        // For non-asset specific table, this loop will break at end of the very first iteration
        foreach ($chunks as $chunk_index => $assetids) {
            $sql = 'SELECT ' . implode(',', $select_fields) . ' FROM ' . $table;
            // For non-asset specific table, "where" condition not is required. We get the whole table in a single go
            if ($asste_specific_table) {
                $sql .= ' WHERE assetid IN (\'' . implode('\',\'', $assetids) . '\')';
            } else {
                if ($table == 'sq_internal_msg') {
                    // Special case for non-asset specific records for 'interal_msg' table
                    // Internal message has 'assetid' field but messages not associated with the asset will have empty assetid
                    $sql .= " WHERE assetid = '' OR assetid IS NULL";
                }
            }
            $results = MatrixDAL::executeSqlAssoc($sql);
            foreach ($results as $record) {
                $table_records_count++;
                $count++;
                if ($count % 10000 == 0) {
                    echo '.';
                }
                // Asset ID associated with this record
                $assetid = $asste_specific_table ? $record['assetid'] : 'n/a';
                // Key field data
                $key_values = array();
                foreach ($key_fields as $key_field) {
                    $temp_key_v = array_get_index($record, $key_field, NULL);
                    if (is_null($temp_key_v)) {
                        // Primary key field must be there
                        continue 2;
                    }
                    $key_values[$key_field] = $temp_key_v;
                }
                //end foreach
                // Original value field data.
                // This is the one we need to check/fix
                $org_values = array();
                foreach ($value_fields as $value_field) {
                    $org_values[$value_field] = array_get_index($record, $value_field, '');
                }
                //end foreach
                // If it's the same in the new and old encodings, that's good.
                foreach ($org_values as $value_field => $value) {
                    $checked = @iconv(SYS_OLD_ENCODING, SYS_NEW_ENCODING . '//IGNORE', $value);
                    if ($value === $checked) {
                        // This field does not requires conversion/checking
                        unset($org_values[$value_field]);
                    }
                }
                //end foreach
                if (empty($org_values)) {
                    // No field values to convert/check
                    continue;
                }
                // Being here means this record contains invalid chars
                $invalid_asset_records[] = array('asset' => $assetid, 'table' => $table, 'keys' => $key_values, 'values' => $org_values);
                $converted_values = array();
                foreach ($org_values as $value_field => $value) {
                    // If not valid, convert the values without igonoring or interprating any chars
                    if (!isValidValue($value)) {
                        // Serialised fields needs to be handled here
                        $serialised_value = FALSE;
                        if ($table == 'sq_ast_attr_val' && $value_field == 'custom_val' && in_array($record['attrid'], $serialise_attrids)) {
                            $serialised_value = TRUE;
                        }
                        if ($table == 'sq_trig' && $value_field == 'data') {
                            $serialised_value = TRUE;
                        }
                        if ($serialised_value) {
                            $us_value = @unserialize($value);
                            if ($us_value === FALSE && serialize(FALSE) !== $value) {
                                // This has invalid serialsed value, but fix it anyway
                                $converted_value = @iconv(SYS_OLD_ENCODING, SYS_NEW_ENCODING . '//IGNORE', $value);
                                // Put this error notice in the script log file
                                $warnings_count++;
                                $msg = 'Serialsed data field "' . $value_field . '" in the table "' . $table . '" (';
                                foreach ($key_values as $field_name => $value) {
                                    $msg .= $field_name . '=' . $value . '; ';
                                }
                                $msg = rtrim($msg, '; ') . ') does not contain unserialisable data. ' . ($reportOnly ? 'Data can still be converted.' : 'Data will be converted anyway.');
                                log_error_msg($msg);
                            } else {
                                if (is_array($us_value)) {
                                    array_walk_recursive($us_value, 'fix_char');
                                    $converted_value = serialize($us_value);
                                } else {
                                    if (is_scalar($us_value)) {
                                        $us_value = @iconv(SYS_OLD_ENCODING, SYS_NEW_ENCODING . '//IGNORE', $us_value);
                                        $converted_value = serialize($us_value);
                                    } else {
                                        $converted_value = $value;
                                    }
                                }
                            }
                        } else {
                            $converted_value = @iconv(SYS_OLD_ENCODING, SYS_NEW_ENCODING . '//IGNORE', $value);
                        }
                        // If the converted value is valid in current encoding then its good to go
                        // otherwise we'll just not use this value
                        if ($converted_value != $value && isValidValue($converted_value)) {
                            $value = $converted_value;
                            $converted_values[$value_field] = $value;
                        }
                    } else {
                        // if it's a valid encoded value, but was convertable before with iconv using old encoding
                        // it might be only because value is already properly encoded with new encoding.  so use md_detect to double check
                        $encoding = mb_detect_encoding($value);
                        if (strtolower($encoding) === strtolower(SYS_NEW_ENCODING)) {
                            unset($org_values[$value_field]);
                        }
                    }
                }
                //end foreach
                if (empty($org_values)) {
                    // All good
                    array_pop($invalid_asset_records);
                    continue;
                }
                // If the successfully converted fields count is same as the invalid fields count, we can proceed with the update
                $update_required = count($org_values) == count($converted_values);
                if ($update_required) {
                    if (!$reportOnly) {
                        $GLOBALS['SQ_SYSTEM']->doTransaction('BEGIN');
                        // Generate update sql
                        $bind_vars = array();
                        $set_sql = array();
                        foreach ($converted_values as $field_name => $value) {
                            $set_sql[] = $field_name . '=:' . $field_name . '_v';
                            $bind_vars[$field_name . '_v'] = $value;
                        }
                        $where_sql = array();
                        foreach ($key_values as $field_name => $value) {
                            $where_sql[] = $field_name . '=:' . $field_name . '_k';
                            $bind_vars[$field_name . '_k'] = $value;
                        }
                        try {
                            $sql = 'UPDATE ' . $table . '
									SET ' . implode(', ', $set_sql) . '
									WHERE ' . implode(' AND ', $where_sql);
                            $update_sql = MatrixDAL::preparePdoQuery($sql);
                            foreach ($bind_vars as $var_name => $var_value) {
                                MatrixDAL::bindValueToPdo($update_sql, $var_name, $var_value);
                            }
                            // Execute the update query
                            $execute = MatrixDAL::executePdoAssoc($update_sql);
                            if (count($execute) > 1) {
                                foreach ($bind_vars as $var_name => $var_value) {
                                    $sql = str_replace(':' . $var_name, "'" . $var_value . "'", $sql);
                                }
                                $errors_count++;
                                $msg = 'Executing query "' . $sql . '" will affect ' . count($execute) . ' records, instead of expected single record! Ignoring this sql.';
                                log_error_msg($msg);
                                $GLOBALS['SQ_SYSTEM']->doTransaction('ROLLBACK');
                            } else {
                                $GLOBALS['SQ_SYSTEM']->doTransaction('COMMIT');
                                $records_fixed_count++;
                                $affected_assetids[$table][] = $assetid;
                            }
                        } catch (Exception $e) {
                            $errors_count++;
                            $msg = "Unexpected error occured while updating database: " . $e->getMessage();
                            log_error_msg($msg);
                            $GLOBALS['SQ_SYSTEM']->doTransaction('ROLLBACK');
                        }
                    } else {
                        $records_fixed_count++;
                        // For reporting purpose only
                        $affected_assetids[$table][] = $assetid;
                    }
                } else {
                    // Trying to carryout charset conversion for this invalid value still resulted into invalid value
                    // Hence record was not updated for this value conversion
                    $errors_count++;
                    $msg = 'Entry in the table "' . $table . '": ' . "\n";
                    foreach ($key_values as $field_name => $field_value) {
                        $msg .= $field_name . '="' . $field_value . '"; ';
                    }
                    $msg .= "\n" . 'contains invalid char(s), which were not replaced because the charset conversion was not successful' . ($msg .= "\n" . 'Potentially invalid characters include:' . listProblematicCharacters($org_values));
                    log_error_msg($msg);
                }
            }
            //end foreach records
            if (!$asste_specific_table) {
                // We have processed all the entries for this non-asset specific table
                break;
            }
        }
        //end foreach assetids chunk
        echo " " . $table_records_count . " records";
    }
    //end foreach tables
    $GLOBALS['SQ_SYSTEM']->restoreDatabaseConnection();
    unset($target_assetids);
    unset($chunks);
    echo "\n";
    $invalid_count = sizeof(array_keys($invalid_asset_records));
    echo "Number of db records with invalid char(s): " . $invalid_count . "\n";
    if ($invalid_count > 0) {
        foreach ($invalid_asset_records as $k => $details) {
            echo "\n\tAsset #" . $details['asset'] . " in table " . $details['table'];
            echo "\n\t" . 'Entry: ';
            foreach ($details['keys'] as $field_name => $field_value) {
                echo $field_name . '="' . $field_value . '"; ';
            }
            echo "\n\tPossibly problematic characters: " . listProblematicCharacters($details['values']) . "\n";
        }
        echo "\n";
    }
    return array('warning_count' => $warnings_count, 'error_count' => $errors_count, 'records_fixed_count' => $records_fixed_count, 'affected_assetids' => $affected_assetids);
}