function detect_bad_utf8($mysqli, $flag = false) { $sql = "select max(id) as total from sc_post "; $row = MySQL\Helper::fetchRow($mysqli, $sql); $total = $row["total"]; $pageSize = 50; $pages = ceil($total / $pageSize); $count = 0; while ($count <= $pages) { $start = $count * $pageSize + 1; $end = $start + ($pageSize - 1); $sql = " select pseudo_id,title,description from sc_post where (id <= {end}) and (id >= {start} ) "; $sql = str_replace(array("{end}", "{start}"), array(0 => $end, 1 => $start), $sql); $rows = MySQL\Helper::fetchRows($mysqli, $sql); printf("processing row between %d and %d \n", $start, $end); foreach ($rows as $row) { $description = $row['description']; if (!Util::isUtf8($description)) { printf("Bad utf-8 for item - %s \n", $row['pseudo_id']); if ($flag) { $clean_description = Util::filterBadUtf8($description); $clean_title = Util::filterBadUtf8($row['title']); update_clean_utf8($mysqli, $row['pseudo_id'], $clean_title, $clean_description); } } } sleep(1); $count++; } }
public function process($prefix, $fieldName) { $this->pipe->process($fieldName); $this->errors = $this->pipe->getErrors(); if (sizeof($this->errors) > 0) { //set errors and return return; } //get meta data and actual file data $this->mediaData = $this->pipe->getMediaData(); $sBlobData = $this->pipe->getFileData(); if (is_null($sBlobData)) { trigger_error('File processing returned Null Data', E_USER_ERROR); } // @todo - check mime type of an image here? // the downside : valid files without mime type would be rejected // image upload is done // clean original file name of malformed utf-8 chars $cleanName = Util::filterBadUtf8($this->mediaData->originalName); if (empty($cleanName)) { trigger_error("image name consists solely of malformed utf-8", E_USER_ERROR); } $this->mediaData->originalName = $cleanName; // now do image specific processings // and create a thumbnail $tBlobData = $this->computeHW($sBlobData); //amazon s3 meta headers $offset = 3600 * 24 * 365; $expiresOn = gmdate('D, d M Y H:i:s \\G\\M\\T', time() + $offset); $headers = array(); $headers["Expires"] = $expiresOn; $headers["Cache-Control"] = "public, max-age=31536000"; $headers["Content-Type"] = $this->mediaData->mime; $storeName = $this->store->persist($prefix, $this->mediaData->originalName, $sBlobData, $headers); // override content-type header for thumbnail // all thumbnails are image type jpeg $headers["Content-Type"] = "image/jpeg"; //change name to jpeg for thumbnail $tname = Util::getThumbnailName($this->mediaData->originalName); $thumbnail = $this->store->persist($prefix, $tname, $tBlobData, $headers); if (is_null($storeName)) { array_push($this->errors, "file storage failed"); return; } $this->mediaData->storeName = $storeName; $this->mediaData->thumbnail = $thumbnail; $this->mediaData->thumbnailName = $tname; if ($this->isS3Store) { $this->mediaData->store = 's3'; $this->mediaData->bucket = Config::getInstance()->get_value("aws.bucket"); } else { $this->mediaData->store = 'local'; //relative URL for local uploads $this->mediaData->bucket = 'media'; } }
static function createPostView($row, $voptions = NULL) { $voptions = empty($voptions) ? array() : $voptions; //default options $options = array(); $options["abbreviate"] = false; $options["image"] = true; $options["group"] = false; //override defaults foreach ($voptions as $key => $value) { $options[$key] = $value; } $imagesJson = $row["images_json"]; $images = json_decode($imagesJson); $view = new \stdClass(); $view->hasImage = false; $view->images = NULL; $view->hasGroups = false; $view->groups = array(); $view->id = $row['id']; $view->itemId = PseudoId::encode($view->id); // title in DB is 128 chars long. // here on page we want to use a 70 char title. // also used in item images alt text // clean up bad utf-8 data for display $view->title = Util::filterBadUtf8($row['title']); $view->title = Util::abbreviate($view->title, 70); $view->description = Util::filterBadUtf8($row['description']); if ($options["abbreviate"]) { $view->description = Util::abbreviate($view->description, 160); } $view->userName = $row['user_name']; $view->createdOn = AppUtil::convertDBTime($row['created_on']); $view->pubUserId = PseudoId::encode($row['login_id']); $view->loginId = $row['login_id']; $view->userPageURI = "/pub/user/" . $view->pubUserId; //process post image. if (!empty($images) && sizeof($images) > 0 && $options["image"]) { /* process image #1 */ $view->hasImage = true; $image = $images[0]; $imgv = self::convertImageJsonObj($image); $view->thumbnail = $imgv["thumbnail"]; $view->height = $imgv["height"]; $view->width = $imgv["width"]; $view->srcImage = $imgv["source"]; /* assign all images */ $view->images = $images; } //process groups if ($options["group"] === true) { $group_slug = $row['group_slug']; $groups = array(); if (!is_null($group_slug) && strlen($group_slug) > 0) { $slugs = explode(Constants::SPACE, $group_slug); $display = NULL; foreach ($slugs as $slug) { if (empty($slug)) { continue; } //@imp @todo @hack // dirty hack - for single quotes in group name - for old data // anything indexed as flury's - should be converted to flury // now we ignore the single quote in group name so we should be fine $slug = str_replace("'s", "", $slug); $display = StringUtil::convertKeyToName($slug); $groups[] = array("slug" => $slug, "display" => $display); } } if (sizeof($groups) > 0) { $view->hasGroups = true; $view->groups = $groups; } } return $view; }
function post_to_activity($mysqli) { $sql = "select max(id) as total from sc_post"; $row = MySQL\Helper::fetchRow($mysqli, $sql); $total = $row["total"]; $pageSize = 50; $pages = ceil($total / $pageSize); $count = 0; $userDao = new \com\indigloo\sc\dao\User(); $activityDao = new \com\indigloo\sc\dao\Activity(); while ($count <= $pages) { $start = $count * $pageSize + 1; $end = $start + ($pageSize - 1); $sql = " select * from sc_post where (id <= {end}) and (id >= {start} ) "; $sql = str_replace(array("{end}", "{start}"), array(0 => $end, 1 => $start), $sql); $rows = MySQL\Helper::fetchRows($mysqli, $sql); foreach ($rows as $row) { $subjectId = $row['login_id']; $ownerId = $row['login_id']; $postId = $row['id']; $objectId = PseudoId::encode($postId); $userDBRow = $userDao->getOnLoginId($subjectId); $subject = $userDBRow['name']; $object = $row['title']; $object = Util::filterBadUtf8($object); $verb = \com\indigloo\sc\Constants::POST_VERB; $activityDao->addRow($ownerId, $subjectId, $objectId, $subject, $object, $verb); } flush(); sleep(1); $count++; } }