public function test_fix_utf8()
{
// Make sure valid data including other types is not changed.
$this->assertSame(null, fix_utf8(null));
$this->assertSame(1, fix_utf8(1));
$this->assertSame(1.1, fix_utf8(1.1));
$this->assertSame(true, fix_utf8(true));
$this->assertSame('', fix_utf8(''));
$this->assertSame('abc', fix_utf8('abc'));
$array = array('do', 're', 'mi');
$this->assertSame($array, fix_utf8($array));
$object = new stdClass();
$object->a = 'aa';
$object->b = 'bb';
$this->assertEquals($object, fix_utf8($object));
// valid utf8 string
$this->assertSame("žlutý koníček přeskočil potůček \n\t\r", fix_utf8("žlutý koníček přeskočil potůček \n\t\r"));
// Invalid utf8 string.
$this->assertSame('aš', fix_utf8('a' . chr(130) . 'š'), 'This fails with buggy iconv() when mbstring extenstion is not available as fallback.');
}
/**
* Makes sure the data is using valid utf8, invalid characters are discarded.
*
* Note: this function is not intended for full objects with methods and private properties.
*
* @param mixed $value
* @return mixed with proper utf-8 encoding
*/
function fix_utf8($value)
{
if (is_null($value) or $value === '') {
return $value;
} else {
if (is_string($value)) {
if ((string) (int) $value === $value) {
// Shortcut.
return $value;
}
// No null bytes expected in our data, so let's remove it.
$value = str_replace("", '', $value);
// Note: this duplicates min_fix_utf8() intentionally.
static $buggyiconv = null;
if ($buggyiconv === null) {
$buggyiconv = (!function_exists('iconv') or @iconv('UTF-8', 'UTF-8//IGNORE', '100' . chr(130) . '€') !== '100€');
}
if ($buggyiconv) {
if (function_exists('mb_convert_encoding')) {
$subst = mb_substitute_character();
mb_substitute_character('');
$result = mb_convert_encoding($value, 'utf-8', 'utf-8');
mb_substitute_character($subst);
} else {
// Warn admins on admin/index.php page.
$result = $value;
}
} else {
$result = @iconv('UTF-8', 'UTF-8//IGNORE', $value);
}
return $result;
} else {
if (is_array($value)) {
foreach ($value as $k => $v) {
$value[$k] = fix_utf8($v);
}
return $value;
} else {
if (is_object($value)) {
// Do not modify original.
$value = clone $value;
foreach ($value as $k => $v) {
$value->{$k} = fix_utf8($v);
}
return $value;
} else {
// This is some other type, no utf-8 here.
return $value;
}
}
}
}
}
function FetchSaveMovie($id, $lookup)
{
$debug = 0;
$video = runSQL('SELECT * FROM ' . TBL_DATA . ' WHERE id = ' . $id);
// get fields (according to list) from db to be saved later
if ($debug) {
echo "\n=================== Video DB Data ============================\n";
print_r($video[0]);
echo "\n=================== Video DB Data ============================\n";
}
$imdbID = $video[0]['imdbID'];
echo "Movie/imdb -- " . $video[0]['title'] . "/" . $video[0]['imdbID'] . "\n";
if (empty($imdbID)) {
echo "No imdbID\n";
return;
}
if (empty($engine)) {
$engine = engineGetEngine($imdbID);
}
if ($debug) {
echo "IMDBID = {$imdbID}, engine = {$engine}\n";
}
$imdbdata = engineGetData($imdbID, $engine);
# removed due to performance issues of is_utf8
// fix erroneous IMDB encoding issues
if (!is_utf8($imdbdata)) {
echo "Applying encoding fix\n";
$imdbdata = fix_utf8($imdbdata);
}
if (empty($imdbdata[title])) {
echo "Fetch failed , try again...\n";
$imdbdata = engineGetData($imdbID, $engine);
}
if (empty($imdbdata[title])) {
echo "Fetch failed again , next movie";
return;
}
if ($debug) {
echo "\n=================== IMDB Data ============================\n";
print_r($imdbdata);
echo "\n=================== IMDB Data ============================\n";
}
if (!empty($imdbdata[title])) {
//
// NOTE: comment out any of the following lines if you do not want them updated
//
$video[0][title] = $imdbdata[title];
$video[0][subtitle] = $imdbdata[subtitle];
$video[0][year] = $imdbdata[year];
$video[0][imgurl] = $imdbdata[coverurl];
$video[0][runtime] = $imdbdata[runtime];
$video[0][director] = $imdbdata[director];
$video[0][rating] = $imdbdata[rating];
$video[0][country] = $imdbdata[country];
$video[0][language] = $imdbdata[language];
$video[0][actors] = $imdbdata[cast];
$video[0][plot] = $imdbdata[plot];
}
if (count($genres) == 0 || $lookup > 1) {
$genres = array();
$gnames = $imdbdata['genres'];
if (isset($gnames)) {
foreach ($gnames as $gname) {
// check if genre is found- otherwise fail silently
if (is_numeric($genre = getGenreId($gname))) {
$genres[] = $genre;
} else {
echo "MISSING GENRE {$gname}\n";
}
}
}
}
// custom filds , not working for now
for ($i = 1; $i <= 4; $i++) {
$custom = 'custom' . $i;
$type = $config[$custom . 'type'];
if (!empty($type)) {
// copy imdb data into corresponding custom field
$video[0][$custom] = $imdbdata[$type];
echo "CUSTOM {$custom} {$type} = {$imdbdata[$type]}\n";
}
}
// -------- SAVE
$SETS = prepareSQL($video[0]);
if ($debug) {
echo "\n=================== Final Data ============================\n";
echo "SETS = {$SETS} \n";
echo "\n=================== Final Data ============================\n";
}
$id = updateDB($SETS, $id);
// save genres
setItemGenres($id, $genres);
// set seen for currently logged in user
set_userseen($id, $seen);
}
/**
* Add unicode flag to all files in archive.
*
* NOTE: single disk archives only, no ZIP64 support.
*
* @return bool success, modifies the file contents
*/
protected function fix_utf8_flags()
{
if ($this->emptyziphack) {
return true;
}
if (!file_exists($this->archivepathname)) {
return true;
}
// Note: the ZIP structure is described at http://www.pkware.com/documents/casestudies/APPNOTE.TXT
if (!($fp = fopen($this->archivepathname, 'rb+'))) {
return false;
}
if (!($filesize = filesize($this->archivepathname))) {
return false;
}
$centralend = self::zip_get_central_end($fp, $filesize);
if ($centralend === false or $centralend['disk'] !== 0 or $centralend['disk_start'] !== 0 or $centralend['offset'] === 0xffffffff) {
// Single disk archives only and o support for ZIP64, sorry.
fclose($fp);
return false;
}
fseek($fp, $centralend['offset']);
$data = fread($fp, $centralend['size']);
$pos = 0;
$files = array();
for ($i = 0; $i < $centralend['entries']; $i++) {
$file = self::zip_parse_file_header($data, $centralend, $pos);
if ($file === false) {
// Wrong header, sorry.
fclose($fp);
return false;
}
$newgeneral = $file['general'] | pow(2, 11);
if ($newgeneral === $file['general']) {
// Nothing to do with this file.
continue;
}
if (preg_match('/^[a-zA-Z0-9_\\-\\.]*$/', $file['name'])) {
// ASCII file names are always ok.
continue;
}
if ($file['extra']) {
// Most probably not created by php zip ext, better to skip it.
continue;
}
if (fix_utf8($file['name']) !== $file['name']) {
// Does not look like a valid utf-8 encoded file name, skip it.
continue;
}
// Read local file header.
fseek($fp, $file['local_offset']);
$localfile = unpack('Vsig/vversion_req/vgeneral/vmethod/vmtime/vmdate/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length', fread($fp, 30));
if ($localfile['sig'] !== 0x4034b50) {
// Borked file!
fclose($fp);
return false;
}
$file['local'] = $localfile;
$files[] = $file;
}
foreach ($files as $file) {
$localfile = $file['local'];
// Add the unicode flag in central file header.
fseek($fp, $file['central_offset'] + 8);
if (ftell($fp) === $file['central_offset'] + 8) {
$newgeneral = $file['general'] | pow(2, 11);
fwrite($fp, pack('v', $newgeneral));
}
// Modify local file header too.
fseek($fp, $file['local_offset'] + 6);
if (ftell($fp) === $file['local_offset'] + 6) {
$newgeneral = $localfile['general'] | pow(2, 11);
fwrite($fp, pack('v', $newgeneral));
}
}
fclose($fp);
return true;
}
/**
* Determine if there is data waiting to be processed from a form
*
* Used on most forms in Moodle to check for data
* Returns the data as an object, if it's found.
* This object can be used in foreach loops without
* casting because it's cast to (array) automatically
*
* Checks that submitted POST data exists and returns it as object.
*
* @return mixed false or object
*/
function data_submitted()
{
if (empty($_POST)) {
return false;
} else {
return (object) fix_utf8($_POST);
}
}
/**
* Makes sure the data is using valid utf8, invalid characters are discarded.
*
* Note: this function is not intended for full objects with methods and private properties.
*
* @param mixed $value
* @return mixed with proper utf-8 encoding
*/
function fix_utf8($value)
{
if (is_null($value) or $value === '') {
return $value;
} else {
if (is_string($value)) {
if ((string) (int) $value === $value) {
// shortcut
return $value;
}
return iconv('UTF-8', 'UTF-8//IGNORE', $value);
} else {
if (is_array($value)) {
foreach ($value as $k => $v) {
$value[$k] = fix_utf8($v);
}
return $value;
} else {
if (is_object($value)) {
$value = clone $value;
// do not modify original
foreach ($value as $k => $v) {
$value->{$k} = fix_utf8($v);
}
return $value;
} else {
// this is some other type, no utf-8 here
return $value;
}
}
}
}
}
/**
* Makes sure the data is using valid utf8, invalid characters are discarded.
*
* Note: this function is not intended for full objects with methods and private properties.
*
* @param mixed $value
* @return mixed with proper utf-8 encoding
*/
function fix_utf8($value)
{
if (is_null($value) or $value === '') {
return $value;
} else {
if (is_string($value)) {
if ((string) (int) $value === $value) {
// shortcut
return $value;
}
// lower error reporting because glibc throws bogus notices
$olderror = error_reporting();
if ($olderror & E_NOTICE) {
error_reporting($olderror ^ E_NOTICE);
}
$result = iconv('UTF-8', 'UTF-8//IGNORE', $value);
if ($olderror & E_NOTICE) {
error_reporting($olderror);
}
return $result;
} else {
if (is_array($value)) {
foreach ($value as $k => $v) {
$value[$k] = fix_utf8($v);
}
return $value;
} else {
if (is_object($value)) {
$value = clone $value;
// do not modify original
foreach ($value as $k => $v) {
$value->{$k} = fix_utf8($v);
}
return $value;
} else {
// this is some other type, no utf-8 here
return $value;
}
}
}
}
}
function test_fix_utf8()
{
// make sure valid data including other types is not changed
$this->assertidentical(null, fix_utf8(null));
$this->assertidentical(1, fix_utf8(1));
$this->assertidentical(1.1, fix_utf8(1.1));
$this->assertidentical(true, fix_utf8(true));
$this->assertidentical('', fix_utf8(''));
$array = array('do', 're', 'mi');
$this->assertidentical($array, fix_utf8($array));
$object = new stdClass();
$object->a = 'aa';
$object->b = 'bb';
$this->assertidentical($object, fix_utf8($object));
// valid utf8 string
$this->assertidentical("žlutý koníček přeskočil potůček \n\t\r", fix_utf8("žlutý koníček přeskočil potůček \n\t\r"));
// invalid utf8 string
$this->assertidentical('aaabbb', fix_utf8('aaa' . chr(130) . 'bbb'));
}
/**
* Makes sure the data is using valid utf8, invalid characters are discarded.
*
* Note: this function is not intended for full objects with methods and private properties.
*
* @param mixed $value
* @return mixed with proper utf-8 encoding
*/
function fix_utf8($value)
{
if (is_null($value) or $value === '') {
return $value;
} else {
if (is_string($value)) {
if ((string) (int) $value === $value) {
// shortcut
return $value;
}
// Lower error reporting because glibc throws bogus notices.
$olderror = error_reporting();
if ($olderror & E_NOTICE) {
error_reporting($olderror ^ E_NOTICE);
}
// Note: this duplicates min_fix_utf8() intentionally.
static $buggyiconv = null;
if ($buggyiconv === null) {
$buggyiconv = (!function_exists('iconv') or iconv('UTF-8', 'UTF-8//IGNORE', '100' . chr(130) . '\\80') !== '100\\80');
}
if ($buggyiconv) {
if (function_exists('mb_convert_encoding')) {
$subst = mb_substitute_character();
mb_substitute_character('');
$result = mb_convert_encoding($value, 'utf-8', 'utf-8');
mb_substitute_character($subst);
} else {
// Warn admins on admin/index.php page.
$result = $value;
}
} else {
$result = iconv('UTF-8', 'UTF-8//IGNORE', $value);
}
if ($olderror & E_NOTICE) {
error_reporting($olderror);
}
return $result;
} else {
if (is_array($value)) {
foreach ($value as $k => $v) {
$value[$k] = fix_utf8($v);
}
return $value;
} else {
if (is_object($value)) {
$value = clone $value;
// do not modify original
foreach ($value as $k => $v) {
$value->{$k} = fix_utf8($v);
}
return $value;
} else {
// this is some other type, no utf-8 here
return $value;
}
}
}
}
}
/**
* Upload program objectives.
* @param object $mform form definition
* @return void
*/
private function upload_program_obj($mform)
{
global $DB, $CFG, $USER;
global $categoryId;
$obj_was_uploaded = $mform->getSubmitValue('upload_program_obj');
if ($obj_was_uploaded) {
$files = $this->get_draft_files('temp_program_obj');
if (!empty($files)) {
$file = reset($files);
$content = $file->get_content();
$all_rows = explode("\n", $content);
$groups = array();
$titles = array();
$type = new stdClass();
$type->typename = $mform->getSubmitValue('new_group');
$type->category = $categoryId;
$masterid = $DB->insert_record('objectivetypes', $type, true);
foreach ($all_rows as $row) {
$parsed = str_getcsv($row);
if (!is_null($parsed[0])) {
if ($parsed[0] != '' && $parsed[1] != '') {
$parsed[0] = fix_utf8($parsed[0]);
$parsed[1] = fix_utf8($parsed[1]);
if (!array_key_exists($parsed[0], $groups)) {
$group = new stdClass();
$group->groupname = $parsed[0];
$group->parent = $masterid;
$groupid = $DB->insert_record('objectivegroups', $group, true);
$groups[$parsed[0]] = $groupid;
// set to ID
}
if (!array_key_exists($parsed[1], $titles)) {
$title = new stdClass();
$title->objectivename = $parsed[1];
$title->parent = null;
$title->objectivegroup = $groups[$parsed[0]];
//echo "writing title\n";
//echo $parsed[1];
$titleid = $DB->insert_record('programobjectives', $title, true);
//echo "wrote title\n";
//$titleid = 1;
$titles[$parsed[1]] = $titleid;
// set to ID
}
if ($parsed[2] != '') {
$parsed[2] = fix_utf8($parsed[2]);
$title = new stdClass();
$title->objectivename = $parsed[2];
$title->parent = $titles[$parsed[1]];
$title->objectivegroup = $groups[$parsed[0]];
$DB->insert_record('programobjectives', $title, false);
}
//$this->insert_program_objective($groups[$parsed[0]], $groups[$parsed[1]], $parsed[2]);
}
}
}
}
}
}
/**
* Converts the text between different encodings. It uses iconv extension with //TRANSLIT parameter,
* falls back to typo3. If both source and target are utf-8 it tries to fix invalid characters only.
*
* @param string $text
* @param string $fromCS source encoding
* @param string $toCS result encoding
* @return string|bool converted string or false on error
*/
public static function convert($text, $fromCS, $toCS = 'utf-8')
{
$fromCS = self::parse_charset($fromCS);
$toCS = self::parse_charset($toCS);
$text = (string) $text;
// we can work only with strings
if ($text === '') {
return '';
}
if ($fromCS === 'utf-8') {
$text = fix_utf8($text);
if ($toCS === 'utf-8') {
return $text;
}
}
if ($toCS === 'ascii') {
// Try to normalize the conversion a bit.
$text = self::specialtoascii($text, $fromCS);
}
// Prevent any error notices, do not use //IGNORE so that we get
// consistent result from Typo3 if iconv fails.
$result = @iconv($fromCS, $toCS . '//TRANSLIT', $text);
if ($result === false or $result === '') {
// note: iconv is prone to return empty string when invalid char encountered, or false if encoding unsupported
$oldlevel = error_reporting(E_PARSE);
$result = self::typo3()->conv((string) $text, $fromCS, $toCS);
error_reporting($oldlevel);
}
return $result;
}
// PayPal does not like when we return error messages here,
// the custom handler just logs exceptions and stops.
set_exception_handler('enrol_paypal_ipn_exception_handler');
/// Keep out casual intruders
if (empty($_POST) or !empty($_GET)) {
print_error("Sorry, you can not use the script that way.");
}
/// Read all the data from PayPal and get it ready for later;
/// we expect only valid UTF-8 encoding, it is the responsibility
/// of user to set it up properly in PayPal business account,
/// it is documented in docs wiki.
$req = 'cmd=_notify-validate';
$data = new stdClass();
foreach ($_POST as $key => $value) {
$req .= "&{$key}=" . urlencode($value);
$data->{$key} = fix_utf8($value);
}
$custom = explode('-', $data->custom);
$data->userid = (int) $custom[0];
$data->courseid = (int) $custom[1];
$data->instanceid = (int) $custom[2];
$data->payment_gross = $data->mc_gross;
$data->payment_currency = $data->mc_currency;
$data->timeupdated = time();
/// get the user and course records
if (!($user = $DB->get_record("user", array("id" => $data->userid)))) {
message_paypal_error_to_admin("Not a valid user id", $data);
die;
}
if (!($course = $DB->get_record("course", array("id" => $data->courseid)))) {
message_paypal_error_to_admin("Not a valid course id", $data);
/**
* Registers the commit info for all new commits on the given branch
*
* @param PHPGit_Repository $repo repository to parse
* @param string $gitbranch the real name of the branch to analyze (eg 'master')
* @param string $branch the future name of the same branch (eg 'MOODLE_28_STABLE')
* @param string $mergemode either 'merges' or 'no-merges'
* @param bool $showprogress
* @internal
*/
function dev_git_record_commits(PHPGit_Repository $repo, $gitbranch, $branch, $mergemode, $showprogress = false)
{
global $DB;
$startpoints = get_config('local_dev', 'gitstartpoints');
if ($startpoints === false) {
set_config('gitstartpoints', json_encode(array()), 'local_dev');
$startpoints = get_config('local_dev', 'gitstartpoints');
}
$startpoints = json_decode($startpoints, true);
$reponame = basename($repo->getDir());
$exclude = empty($startpoints[$branch][$mergemode]) ? '' : $startpoints[$branch][$mergemode];
if ($mergemode === 'merges') {
fputs(STDOUT, "Searching merges on {$gitbranch} ({$branch})" . ($exclude ? " from {$exclude}" : "") . PHP_EOL);
$mergeflag = 1;
} else {
if ($mergemode === 'no-merges') {
fputs(STDOUT, "Searching non-merges on {$gitbranch} ({$branch})" . ($exclude ? " from {$exclude}" : "") . PHP_EOL);
$mergeflag = 0;
}
}
$exclude = empty($exclude) ? '' : '^' . $exclude;
$commits = explode(PHP_EOL, $repo->git("rev-list --reverse --{$mergemode} --format='tformat:COMMIT:%H TIMESTAMP:%at AUTHORNAME:%an AUTHOREMAIL:%ae SUBJECT:%s' {$gitbranch} {$exclude}"));
$total = floor(count($commits) / 2);
$counter = 0;
if ($showprogress and $total == 0) {
fputs(STDOUT, 'no commits found');
}
foreach ($commits as $commit) {
$pattern = '/^COMMIT:([0-9a-f]{40}) TIMESTAMP:([0-9]+) AUTHORNAME:(.+) AUTHOREMAIL:(.+) SUBJECT:(.*)$/';
if (!preg_match($pattern, $commit, $matches)) {
continue;
}
$record = new stdClass();
$record->repository = $reponame;
$record->commithash = $matches[1];
$record->authordate = $matches[2];
$record->authorname = $matches[3];
$record->authoremail = $matches[4];
$record->subject = $matches[5];
$record->merge = $mergeflag;
$record = @fix_utf8($record);
// register the commit info record if it does not exist yet
$existing = $DB->get_record('dev_git_commits', array('repository' => $reponame, 'commithash' => $record->commithash), 'id', IGNORE_MISSING);
if ($existing === false) {
$commitid = $DB->insert_record('dev_git_commits', $record, true, true);
} else {
$commitid = $existing->id;
}
// register the branch containing the current commit
if (!$DB->record_exists('dev_git_commit_branches', array('branch' => $branch, 'commitid' => $commitid))) {
$branchinfo = new stdClass();
$branchinfo->commitid = $commitid;
$branchinfo->branch = $branch;
$DB->insert_record('dev_git_commit_branches', $branchinfo, false, true);
}
if ($showprogress) {
fputs(STDOUT, ++$counter . '/' . $total . "\r");
}
$startpoints[$branch][$mergemode] = $record->commithash;
if ($counter % 1000 == 0) {
set_config('gitstartpoints', json_encode($startpoints), 'local_dev');
}
}
set_config('gitstartpoints', json_encode($startpoints), 'local_dev');
if ($showprogress) {
fputs(STDOUT, PHP_EOL);
}
}
/**
* Converts the text between different encodings. It uses iconv extension with //TRANSLIT parameter,
* falls back to typo3. If both source and target are utf-8 it tries to fix invalid characters only.
*
* @param string $text
* @param string $fromCS source encoding
* @param string $toCS result encoding
* @return string|bool converted string or false on error
*/
public static function convert($text, $fromCS, $toCS='utf-8') {
$fromCS = self::parse_charset($fromCS);
$toCS = self::parse_charset($toCS);
$text = (string)$text; // we can work only with strings
if ($text === '') {
return '';
}
if ($toCS === 'utf-8' and $fromCS === 'utf-8') {
return fix_utf8($text);
}
$result = iconv($fromCS, $toCS.'//TRANSLIT', $text);
if ($result === false or $result === '') {
// note: iconv is prone to return empty string when invalid char encountered, or false if encoding unsupported
$oldlevel = error_reporting(E_PARSE);
$result = self::typo3()->conv((string)$text, $fromCS, $toCS);
error_reporting($oldlevel);
}
return $result;
}
