/** * Process an aggregate-type bucket. This is MySQL specific. * * @param Plugins_DeliveryLog $oBucket a reference to the using (context) object. * @param Date $oEnd A PEAR_Date instance, interval_start to process up to (inclusive). */ public function processBucket($oBucket, $oEnd) { $sTableName = $oBucket->getBucketTableName(); $oMainDbh =& OA_DB_Distributed::singleton(); if (PEAR::isError($oMainDbh)) { MAX::raiseError($oMainDbh, MAX_ERROR_DBFAILURE, PEAR_ERROR_DIE); } OA::debug(' - Processing the ' . $sTableName . ' table for data with operation interval start equal to or before ' . $oEnd->format('%Y-%m-%d %H:%M:%S') . ' ' . $oEnd->tz->getShortName(), PEAR_LOG_INFO); // Select all rows with interval_start <= previous OI start. $rsData =& $this->getBucketTableContent($sTableName, $oEnd); $rowCount = $rsData->getRowCount(); OA::debug(' - ' . $rsData->getRowCount() . ' records found', PEAR_LOG_DEBUG); if ($rowCount) { // We can't do bulk inserts with ON DUPLICATE. $aExecQueries = array(); while ($rsData->fetch()) { // Get first row $aRow = $rsData->toArray(); // Insert or update $aExecQueries[] = "SELECT bucket_update_{$sTableName}(" . join(',', array_map(array(&$oMainDbh, 'quote'), $aRow)) . ")"; } if (count($aExecQueries)) { foreach ($aExecQueries as $execQuery) { $result = $oMainDbh->exec($execQuery); if (PEAR::isError($result)) { MAX::raiseError($result, MAX_ERROR_DBFAILURE, PEAR_ERROR_DIE); } } } } }
/** * A method to return a singleton database connection resource. * * Example usage: * $oDbh = OA_DB_Distributed::singleton(); * * Warning: In order to work correctly, the singleton method must * be instantiated statically and by reference, as in the above * example. * * @static * @param string $dsn Optional database DSN details - connects to the * database defined by the configuration file otherwise. * See {@link OA_DB::getDsn()} for format. * @return MDB2_Driver_Common An MDB2 connection resource, or PEAR_Error * on failure to connect. */ static function singleton($dsn = null) { // Get the DSN, if not set $dsn = is_null($dsn) ? OA_DB_Distributed::getDsn() : $dsn; // Should the connection have options set? $aDriverOptions = OA_DB_Distributed::getDsnOptions(); // Return the datbase connection return parent::singleton($dsn, $aDriverOptions); }
/** * Process a raw-type bucket. * * @param Plugins_DeliveryLog a reference to the using (context) object. * @param Date $oEnd A PEAR_Date instance, interval_start to process up to (inclusive). */ public function processBucket($oBucket, $oEnd) { $sTableName = $oBucket->getBucketTableName(); $oMainDbh =& OA_DB_Distributed::singleton(); if (PEAR::isError($oMainDbh)) { MAX::raiseError($oMainDbh, MAX_ERROR_DBFAILURE, PEAR_ERROR_DIE); } OA::debug(' - Processing the ' . $sTableName . ' table for data with operation interval start equal to or before ' . $oEnd->format('%Y-%m-%d %H:%M:%S') . ' ' . $oEnd->tz->getShortName(), PEAR_LOG_INFO); // As this is raw data being processed, data will not be logged based on the operation interval, // but based on the time the raw data was collected. Adjust the $oEnd value accordingly... $aDates = OX_OperationInterval::convertDateToOperationIntervalStartAndEndDates($oEnd); OA::debug(' - The ' . $sTableName . ' table is a raw data table. Data logged in real-time, not operation intervals.', PEAR_LOG_INFO); OA::debug(' - Accordingly, processing of the ' . $sTableName . ' table will be performed based on data that has a logged date equal to', PEAR_LOG_INFO); OA::debug(' or before ' . $aDates['end']->format('%Y-%m-%d %H:%M:%S') . ' ' . $aDates['end']->tz->getShortName(), PEAR_LOG_INFO); // Select all rows with interval_start <= previous OI start. $rsData =& $this->getBucketTableContent($sTableName, $aDates['end']); $count = $rsData->getRowCount(); OA::debug(' - ' . $rsData->getRowCount() . ' records found', PEAR_LOG_DEBUG); if ($count) { $packetSize = 16777216; // 16 MB hardcoded (there's no max limit) $i = 0; while ($rsData->fetch()) { $aRow = $rsData->toArray(); $sRow = '(' . join(',', array_map(array(&$oMainDbh, 'quote'), $aRow)) . ')'; if (!$i) { $sInsert = "INSERT INTO {$sTableName} (" . join(',', array_keys($aRow)) . ") VALUES "; $query = ''; $aExecQueries = array(); } if (!$query) { $query = $sInsert . $sRow; // Leave 4 bytes headroom for max_allowed_packet } elseif (strlen($query) + strlen($sRow) + 4 < $packetSize) { $query .= ',' . $sRow; } else { $aExecQueries[] = $query; $query = $sInsert . $sRow; } if (++$i >= $count || strlen($query) >= $packetSize) { $aExecQueries[] = $query; $query = ''; } if (count($aExecQueries)) { foreach ($aExecQueries as $execQuery) { $result = $oMainDbh->exec($execQuery); if (PEAR::isError($result)) { MAX::raiseError($result, MAX_ERROR_DBFAILURE, PEAR_ERROR_DIE); } } $aExecQueries = array(); } } } }
/** * Process an aggregate-type bucket. This is MySQL specific. * * @param Plugins_DeliveryLog $oBucket a reference to the using (context) object. * @param Date $oEnd A PEAR_Date instance, interval_start to process up to (inclusive). */ public function processBucket($oBucket, $oEnd) { $sTableName = $oBucket->getBucketTableName(); $oMainDbh =& OA_DB_Distributed::singleton(); if (PEAR::isError($oMainDbh)) { MAX::raiseError($oMainDbh, MAX_ERROR_DBFAILURE, PEAR_ERROR_DIE); } OA::debug(' - Processing the ' . $sTableName . ' table for data with operation interval start equal to or before ' . $oEnd->format('%Y-%m-%d %H:%M:%S') . ' ' . $oEnd->tz->getShortName(), PEAR_LOG_INFO); // Select all rows with interval_start <= previous OI start. $rsData =& $this->getBucketTableContent($sTableName, $oEnd); $rowCount = $rsData->getRowCount(); OA::debug(' - ' . $rsData->getRowCount() . ' records found', PEAR_LOG_DEBUG); if ($rowCount) { // We can't do bulk inserts with ON DUPLICATE. $aExecQueries = array(); if ($rsData->fetch()) { // Get first row $aRow = $rsData->toArray(); // Prepare INSERT $sInsert = "INSERT INTO {$sTableName} (" . join(',', array_keys($aRow)) . ") VALUES "; // Add first row data $sRow = '(' . join(',', array_map(array(&$oMainDbh, 'quote'), $aRow)) . ')'; $sOnDuplicate = ' ON DUPLICATE KEY UPDATE count = count + ' . $aRow['count']; // Add first insert $aExecQueries[] = $sInsert . $sRow . $sOnDuplicate; // Deal with the other rows while ($rsData->fetch()) { $aRow = $rsData->toArray(); $sRow = '(' . join(',', array_map(array(&$oMainDbh, 'quote'), $aRow)) . ')'; $sOnDuplicate = ' ON DUPLICATE KEY UPDATE count = count + ' . $aRow['count']; $aExecQueries[] = $sInsert . $sRow . $sOnDuplicate; } } if (count($aExecQueries)) { // Try to disable the binlog for the inserts so we don't // replicate back out over our logged data. PEAR::staticPushErrorHandling(PEAR_ERROR_RETURN); $result = $oMainDbh->exec('SET SQL_LOG_BIN = 0'); if (PEAR::isError($result)) { OA::debug('Unable to disable the bin log, proceeding anyway.', PEAR_LOG_WARNING); } PEAR::staticPopErrorHandling(); foreach ($aExecQueries as $execQuery) { $result = $oMainDbh->exec($execQuery); if (PEAR::isError($result)) { MAX::raiseError($result, MAX_ERROR_DBFAILURE, PEAR_ERROR_DIE); } } } } }
/** * Process a raw-type bucket. * * @param Plugins_DeliveryLog a reference to the using (context) object. * @param Date $oEnd A PEAR_Date instance, interval_start to process up to (inclusive). */ public function processBucket($oBucket, $oEnd) { $sTableName = $oBucket->getBucketTableName(); $oMainDbh =& OA_DB_Distributed::singleton(); if (PEAR::isError($oMainDbh)) { MAX::raiseError($oMainDbh, MAX_ERROR_DBFAILURE, PEAR_ERROR_DIE); } OA::debug(' - Processing the ' . $sTableName . ' table for data with operation interval start equal to or before ' . $oEnd->format('%Y-%m-%d %H:%M:%S') . ' ' . $oEnd->tz->getShortName(), PEAR_LOG_INFO); // As this is raw data being processed, data will not be logged based on the operation interval, // but based on the time the raw data was collected. Adjust the $oEnd value accordingly... $aDates = OX_OperationInterval::convertDateToOperationIntervalStartAndEndDates($oEnd); OA::debug(' - The ' . $sTableName . ' table is a raw data table. Data logged in real-time, not operation intervals.', PEAR_LOG_INFO); OA::debug(' - Accordingly, processing of the ' . $sTableName . ' table will be performed based on data that has a logged date equal to', PEAR_LOG_INFO); OA::debug(' or before ' . $aDates['end']->format('%Y-%m-%d %H:%M:%S') . ' ' . $aDates['end']->tz->getShortName(), PEAR_LOG_INFO); // Select all rows with interval_start <= previous OI start. $rsData =& $this->getBucketTableContent($sTableName, $aDates['end']); $count = $rsData->getRowCount(); OA::debug(' - ' . $rsData->getRowCount() . ' records found', PEAR_LOG_DEBUG); if ($count) { $aRow = $oMainDbh->queryRow("SHOW VARIABLES LIKE 'max_allowed_packet'"); $packetSize = !empty($aRow['value']) ? $aRow['value'] : 0; $i = 0; while ($rsData->fetch()) { $aRow = $rsData->toArray(); $sRow = '(' . join(',', array_map(array(&$oMainDbh, 'quote'), $aRow)) . ')'; if (!$i) { $sInsert = "INSERT INTO {$sTableName} (" . join(',', array_keys($aRow)) . ") VALUES "; $query = ''; $aExecQueries = array(); } if (!$query) { $query = $sInsert . $sRow; // Leave 4 bytes headroom for max_allowed_packet } elseif (strlen($query) + strlen($sRow) + 4 < $packetSize) { $query .= ',' . $sRow; } else { $aExecQueries[] = $query; $query = $sInsert . $sRow; } if (++$i >= $count || strlen($query) >= $packetSize) { $aExecQueries[] = $query; $query = ''; } if (count($aExecQueries)) { // Try to disable the binlog for the inserts so we don't // replicate back out over our logged data. PEAR::staticPushErrorHandling(PEAR_ERROR_RETURN); $result = $oMainDbh->exec('SET SQL_LOG_BIN = 0'); if (PEAR::isError($result)) { OA::debug('Unable to disable the bin log, proceeding anyway.', PEAR_LOG_WARNING); } PEAR::staticPopErrorHandling(); foreach ($aExecQueries as $execQuery) { $result = $oMainDbh->exec($execQuery); if (PEAR::isError($result)) { MAX::raiseError($result, MAX_ERROR_DBFAILURE, PEAR_ERROR_DIE); } } $aExecQueries = array(); } } } }