public static function Init()
{
self::$LOG = 'conversion-queue.log';
self::$TABLE = 'tbx_conversion_queue';
self::$SCRIPT = 'ConversionQueue.php';
self::$CACHE_STATS = 'conversion-queue-stats';
self::$CACHE_PID = 'conversion-queue-pid';
self::$CACHE_STOP = 'conversion-queue-stop';
}
#### Create database tables =======================================================================================================
// Create tbx_imported if doesn't already exist
$DB->Update(GetDBCreate('tbx_imported'));
// Create tbx_thumb_queue if doesn't already exist
$DB->Update(GetDBCreate('tbx_thumb_queue'));
// Create tbx_search_term_new if doesn't already exist
$DB->Update(GetDBCreate('tbx_search_term_new'));
#### Create database tables =======================================================================================================
#### Update database ==============================================================================================================
// Prepare for new QueueProcessor stats format
$stats = ThumbQueue::LoadStats();
if (!isset($stats[ThumbQueue::STAT_PROCESSED_ITEMS])) {
Cache_MySQL::Remove('thumb-queue-stats');
}
// Prepare for new QueueProcessor stats format
$stats = ConversionQueue::LoadStats();
if (!isset($stats[ConversionQueue::STAT_PROCESSED_ITEMS])) {
Cache_MySQL::Remove('conversion-queue-stats');
}
// Update sponsor video counts
if ($DB->QuerySingleColumn('SELECT MAX(`videos`) FROM `tbx_sponsor`') == 0) {
UpdateSponsorStats();
}
// Update status in tbx_video_feed
$DB->Update("ALTER TABLE `tbx_video_feed` MODIFY `status` ENUM('Pending','Scheduled','Active','Disabled') NOT NULL");
// Add image_id to tbx_category
$columns = $DB->GetColumns('tbx_category');
if (!in_array('image_id', $columns)) {
$DB->Update('ALTER TABLE `tbx_category` ADD COLUMN `image_id` INT UNSIGNED');
}
// Add image_id to tbx_video_feed
<?php
echo ResizeableColumn('Videos Processed', $stats[ConversionQueue::STAT_PROCESSED_ITEMS], true);
?>
<?php
echo ResizeableColumn('Failed', $stats[ConversionQueue::STAT_FAILED_ITEMS], true);
?>
<?php
echo ResizeableColumn('Average Time', $stats[ConversionQueue::STAT_AVERAGE_TIME]);
?>
<?php
echo ResizeableColumn('Average Wait', $stats[ConversionQueue::STAT_AVERAGE_WAIT]);
?>
<script language="JavaScript" type="text/javascript">
$(function()
{
$('#dialog-button-start').attr('disabled', '<?php
echo ConversionQueue::IsStartable() ? '' : 'disabled';
?>
');
$('#dialog-button-stop').attr('disabled', '<?php
echo ConversionQueue::IsRunning() ? '' : 'disabled';
?>
');
$('#dialog-button-clear').attr('disabled', '<?php
echo !ConversionQueue::IsRunning() ? '' : 'disabled';
?>
');
});
</script>
function tbxUploadStepTwo()
{
global $t;
$upload = $_FILES['video_file'];
$v = Validator::Create();
$DB = GetDB();
$v->Register(sha1($_REQUEST['step_one_data'] . Config::Get('random_value')) == $_REQUEST['step_one_sig'], Validator_Type::IS_TRUE, _T('Validation:Video Data Altered'));
$v->Register($upload['error'] == UPLOAD_ERR_OK, Validator_Type::IS_TRUE, Uploads::CodeToMessage($upload['error']));
if (is_uploaded_file($upload['tmp_name'])) {
$max_filesize = Format::StringToBytes(Config::Get('max_upload_size'));
$max_duration = Format::DurationToSeconds(Config::Get('max_upload_duration'));
$extensions = str_replace(',', '|', Config::Get('upload_extensions'));
$v->Register($upload['size'], Validator_Type::IS_BETWEEN, _T('Validation:Video size too large'), '1,' . $max_filesize);
$v->Register(File::Extension($upload['name']), Validator_Type::REGEX_MATCH, _T('Validation:Video file extension not allowed'), '~^(' . $extensions . ')$~');
try {
$vi = new Video_Info($upload['tmp_name']);
$vi->Extract();
$v->Register($vi->length, Validator_Type::LESS_EQ, _T('Validation:Video duration too long'), $max_duration);
} catch (Exception $e) {
$v->Register(false, Validator_Type::IS_TRUE, $e->getMessage());
}
$md5 = md5_file($upload['tmp_name']);
if (Config::Get('flag_upload_reject_duplicates')) {
$v->Register($DB->QueryCount('SELECT COUNT(*) FROM `tbx_video_md5sum` WHERE `md5`=?', array($md5)), Validator_Type::IS_ZERO, _T('Validation:Duplicate video'));
}
}
// Validate input
if (!$v->Validate()) {
$t->Assign('g_errors', $v->GetErrors());
$t->AssignByRef('g_form', $_REQUEST);
if (isset($_REQUEST['flash'])) {
$t->Display('upload-flash-errors.tpl');
} else {
$t->Assign('g_file_types', '*.' . str_replace(',', ';*.', Config::Get('upload_extensions')));
$t->Assign('g_cookie', $_COOKIE[LOGIN_COOKIE]);
$t->Display('upload-step-two.tpl');
}
return;
}
$_REQUEST = array_merge($_REQUEST, unserialize(base64_decode($_REQUEST['step_one_data'])));
Form_Prepare::Standard('tbx_video');
Form_Prepare::Standard('tbx_video_stat');
Form_Prepare::Custom('tbx_video_custom_schema', 'on_submit');
$_REQUEST['duration'] = $vi->length;
$_REQUEST['date_added'] = Database_MySQL::Now();
$_REQUEST['username'] = AuthenticateUser::GetUsername();
$_REQUEST['is_private'] = Config::Get('flag_upload_allow_private') ? intval($_REQUEST['is_private']) : 0;
$_REQUEST['allow_ratings'] = intval($_REQUEST['allow_ratings']);
$_REQUEST['allow_embedding'] = intval($_REQUEST['allow_embedding']);
$_REQUEST['allow_comments'] = intval($_REQUEST['allow_comments']) ? 'Yes - Add Immediately' : 'No';
$_REQUEST['is_user_submitted'] = 1;
if ($_REQUEST['recorded_day'] && $_REQUEST['recorded_month'] && $_REQUEST['recorded_year']) {
$_REQUEST['date_recorded'] = $_REQUEST['recorded_year'] . '-' . $_REQUEST['recorded_month'] . '-' . $_REQUEST['recorded_day'];
}
// Strip HTML tags
if (Config::Get('flag_video_strip_tags')) {
$_REQUEST = String::StripTags($_REQUEST);
}
// Configure status
$_REQUEST['status'] = STATUS_ACTIVE;
if (Config::Get('flag_upload_convert')) {
$_REQUEST['status'] = STATUS_QUEUED;
$_REQUEST['next_status'] = Config::Get('flag_upload_review') ? STATUS_PENDING : STATUS_ACTIVE;
} else {
if (Config::Get('flag_upload_review')) {
$_REQUEST['status'] = STATUS_PENDING;
}
}
// Add to database
$_REQUEST['video_id'] = DatabaseAdd('tbx_video', $_REQUEST);
DatabaseAdd('tbx_video_custom', $_REQUEST);
DatabaseAdd('tbx_video_stat', $_REQUEST);
if ($_REQUEST['status'] == STATUS_ACTIVE && !$_REQUEST['is_private']) {
Tags::AddToFrequency($_REQUEST['tags']);
} else {
if ($_REQUEST['status'] == STATUS_QUEUED) {
DatabaseAdd('tbx_conversion_queue', array('video_id' => $_REQUEST['video_id'], 'queued' => time()));
}
}
// Mark as private
if ($_REQUEST['is_private']) {
$_REQUEST['private_id'] = sha1(uniqid(rand(), true));
DatabaseAdd('tbx_video_private', $_REQUEST);
}
// Setup video files and generate thumbnails
$directory = Video_Dir::DirNameFromId($_REQUEST['video_id']);
$vd = new Video_Dir($directory);
$clip = $vd->AddClipFromFile($upload['tmp_name'], File::Extension($upload['name']));
if (Video_FrameGrabber::CanGrab()) {
Video_FrameGrabber::Grab($clip, $vd->GetThumbsDir(), Config::Get('thumb_amount'), Config::Get('thumb_quality'), Config::Get('thumb_size'), $vi);
}
foreach ($vd->GetClipURIs() as $clip) {
$_REQUEST['clip'] = $clip;
$_REQUEST['filesize'] = filesize(Config::Get('document_root') . $clip);
DatabaseAdd('tbx_video_clip', $_REQUEST);
}
$thumb_ids = array();
foreach ($vd->GetThumbURIs() as $thumb) {
$_REQUEST['thumbnail'] = $thumb;
$thumb_ids[] = DatabaseAdd('tbx_video_thumbnail', $_REQUEST);
}
// Select the display thumbnail
$num_thumbnails = count($thumb_ids);
$display_thumbnail = null;
if ($num_thumbnails > 0) {
$display_thumbnail = $thumb_ids[rand(0, floor(0.4 * $num_thumbnails))];
}
DatabaseUpdate('tbx_video', array('video_id' => $_REQUEST['video_id'], 'num_thumbnails' => $num_thumbnails, 'display_thumbnail' => $display_thumbnail));
// Add MD5 sum for prevention of duplicates
$DB->Update('REPLACE INTO `tbx_video_md5sum` VALUES (?)', array($md5));
// Update user stats
StatsRollover();
$DB->Update('UPDATE `tbx_user_stat` SET ' . '`today_videos_uploaded`=`today_videos_uploaded`+1,' . '`week_videos_uploaded`=`week_videos_uploaded`+1,' . '`month_videos_uploaded`=`month_videos_uploaded`+1,' . '`total_videos_uploaded`=`total_videos_uploaded`+1 ' . 'WHERE `username`=?', array($_REQUEST['username']));
$t->AssignByRef('g_form', $_REQUEST);
$t->AssignByRef('g_video', $_REQUEST);
$t->Display(isset($_REQUEST['flash']) ? 'upload-flash-complete.tpl' : 'upload-complete.tpl');
UpdateCategoryStats($_REQUEST['category_id']);
if (!Config::Get('flag_using_cron') && $_REQUEST['status'] == STATUS_QUEUED) {
ConversionQueue::Start();
}
}
function tbxConversionQueueStart()
{
ConversionQueue::Start();
JSON::Success('The conversion queue has been started');
}
public static function Import($settings)
{
$DB = GetDB();
ProgressBarShow('pb-import');
$file = TEMP_DIR . '/' . File::Sanitize($settings['import_file']);
$fp = fopen($file, 'r');
$filesize = filesize($file);
$line = $read = $imported = 0;
$expected = count($settings['fields']);
while (!feof($fp)) {
$line++;
$string = fgets($fp);
$read += strlen($string);
$data = explode($settings['delimiter'], trim($string));
ProgressBarUpdate('pb-import', $read / $filesize * 100);
// Line does not have the expected number of fields
if (count($data) != $expected) {
continue;
}
$video = array();
$defaults = array('category_id' => $settings['category_id'], 'sponsor_id' => $settings['sponsor_id'], 'username' => $settings['username'], 'duration' => Format::DurationToSeconds($settings['duration']), 'status' => $settings['status'], 'next_status' => $settings['status'], 'allow_comments' => $settings['allow_comments'], 'allow_ratings' => $settings['allow_ratings'], 'allow_embedding' => $settings['allow_embedding'], 'is_private' => $settings['is_private'], 'date_added' => Database_MySQL::Now(), 'is_featured' => 0, 'is_user_submitted' => 0, 'conversion_failed' => 0, 'tags' => null, 'title' => null, 'description' => null);
foreach ($settings['fields'] as $index => $field) {
if (!empty($field)) {
$video[$field] = trim($data[$index]);
}
}
// Setup clips
$clips = array();
$thumbs = array();
$clip_type = 'URL';
if (isset($video['embed_code'])) {
// Cannot convert or thumbnail from embed code
$settings['flag_convert'] = $settings['flag_thumb'] = false;
$clips[] = $video['embed_code'];
$clip_type = 'Embed';
} else {
if (isset($video['gallery_url'])) {
$http = new HTTP();
if (!$http->Get($video['gallery_url'])) {
// Broken gallery URL, continue
continue;
}
list($thumbs, $clips) = Video_Source_Gallery::ExtractUrls($http->url, $http->body);
} else {
if (!isset($video['video_url']) && isset($video['base_video_url'])) {
if (!preg_match('~/$~', $video['base_video_url'])) {
$video['base_video_url'] .= '/';
}
foreach (explode(',', $video['video_filename']) as $filename) {
$clips[] = $video['base_video_url'] . $filename;
}
} else {
$clips[] = $video['video_url'];
}
}
}
// Check for duplicate clips
$duplicate = false;
foreach ($clips as $clip) {
if (!Request::Get('flag_skip_imported_check') && $DB->QueryCount('SELECT COUNT(*) FROM `tbx_imported` WHERE `video_url`=?', array($clip)) > 0) {
$duplicate = true;
}
$DB->Update('REPLACE INTO `tbx_imported` VALUES (?)', array($clip));
}
// Dupe found
if ($duplicate) {
continue;
}
// Setup thumbs
if (!isset($video['gallery_url']) && !isset($video['thumbnail_url']) && isset($video['base_thumbnail_url'])) {
if (!preg_match('~/$~', $video['base_thumbnail_url'])) {
$video['base_thumbnail_url'] .= '/';
}
foreach (explode(',', String::FormatCommaSeparated($video['thumbnail_filename'])) as $filename) {
$thumbs[] = $video['base_thumbnail_url'] . $filename;
}
} else {
if (!isset($video['gallery_url']) && isset($video['thumbnail_url'])) {
$thumbs[] = $video['thumbnail_url'];
}
}
// Setup duration
if (isset($video['duration_seconds'])) {
$video['duration'] = $video['duration_seconds'];
} else {
if (isset($video['duration_formatted'])) {
$video['duration'] = Format::DurationToSeconds($video['duration_formatted']);
}
}
// Use description for title
if (empty($video['title'])) {
$video['title'] = isset($video['description']) ? $video['description'] : '';
}
// Use title for description
if (empty($video['description'])) {
$video['description'] = isset($video['title']) ? $video['title'] : '';
}
// Use title for tags
if (empty($video['tags'])) {
$video['tags'] = isset($video['title']) ? $video['title'] : '';
}
// Setup category
if (isset($video['category']) && ($category_id = $DB->QuerySingleColumn('SELECT `category_id` FROM `tbx_category` WHERE `name` LIKE ?', array($video['category']))) !== false) {
$video['category_id'] = $category_id;
} else {
if (($category_id = GetBestCategory($video['title'] . ' ' . $video['description'])) !== null) {
$video['category_id'] = $category_id;
}
}
// Merge in the defaults
$video = array_merge($defaults, $video);
// Format tags and convert to UTF-8
$video['tags'] = Tags::Format($video['tags']);
$video = String::ToUTF8($video);
if (Request::Get('flag_convert')) {
$video['status'] = STATUS_QUEUED;
}
// Add to database
$video['video_id'] = DatabaseAdd('tbx_video', $video);
DatabaseAdd('tbx_video_custom', $video);
DatabaseAdd('tbx_video_stat', $video);
if ($video['is_private']) {
$video['private_id'] = sha1(uniqid(mt_rand(), true));
DatabaseAdd('tbx_video_private', $video);
}
if ($video['status'] == STATUS_QUEUED) {
$video['queued'] = time();
DatabaseAdd('tbx_conversion_queue', $video);
}
if (Request::Get('flag_thumb')) {
$video['queued'] = time();
DatabaseAdd('tbx_thumb_queue', $video);
}
if ($video['status'] == STATUS_ACTIVE && !$video['is_private']) {
Tags::AddToFrequency($video['tags']);
}
// Add clips
foreach ($clips as $clip) {
DatabaseAdd('tbx_video_clip', array('video_id' => $video['video_id'], 'type' => $clip_type, 'clip' => $clip));
}
$dir = new Video_Dir(Video_Dir::DirNameFromId($video['video_id']));
// Process thumbs
$thumb_ids = array();
foreach ($thumbs as $thumb) {
$http = new HTTP();
if ($http->Get($thumb, $thumb)) {
if (Video_Thumbnail::CanResize()) {
$thumb_temp = $dir->AddTempFromVar($http->body, 'jpg');
$thumb_file = Video_Thumbnail::Resize($thumb_temp, Config::Get('thumb_size'), Config::Get('thumb_quality'), $dir->GetThumbsDir());
} else {
$thumb_file = $dir->AddThumbFromVar($http->body);
}
if (!empty($thumb_file)) {
$thumb_ids[] = DatabaseAdd('tbx_video_thumbnail', array('video_id' => $video['video_id'], 'thumbnail' => str_replace(Config::Get('document_root'), '', $thumb_file)));
}
}
}
// Determine number of thumbnails and select random display thumbnail
$num_thumbnails = count($thumb_ids);
$display_thumbnail = null;
if ($num_thumbnails > 0) {
// Select display thumbnail randomly from the first 40%
$display_thumbnail = $thumb_ids[rand(0, floor(0.4 * $num_thumbnails))];
}
DatabaseUpdate('tbx_video', array('video_id' => $video['video_id'], 'num_thumbnails' => $num_thumbnails, 'display_thumbnail' => $display_thumbnail));
$imported++;
}
fclose($fp);
UpdateCategoryStats();
UpdateSponsorStats($settings['sponsor_id']);
$t = new Template();
$t->ClearCache('categories.tpl');
ProgressBarHide('pb-import', NumberFormatInteger($imported) . ' videos have been imported!');
// Start up the thumbnail and converson queues if needed
if (!Config::Get('flag_using_cron')) {
if (Request::Get('flag_convert')) {
ConversionQueue::Start();
}
if (Request::Get('flag_thumb')) {
ThumbQueue::Start();
}
}
File::Delete($file);
}
if (!isset($args['file']) || empty($args['file'])) {
echo "Please specify a filename using the --file option\n";
exit;
}
$DB = GetDB();
$DB->RestoreTables($args['file']);
break;
case '--feed-read':
ReadFromFields();
break;
case '--stats-rollover':
StatsRollover(true);
break;
case '--process-queue':
case '--run-convert-queue':
ConversionQueue::Start();
break;
case '--run-thumb-queue':
ThumbQueue::Start();
break;
case '--activate-scheduled':
ActivateScheduledVideos();
break;
}
function ActivateScheduledVideos()
{
$DB = GetDB();
$args = ParseCommandLine();
$queries = array();
if (!isset($args['sort']) || empty($args['sort'])) {
$args['sort'] = 'RAND()';
public function Import()
{
$imported = 0;
$http = new HTTP();
if ($http->Get($this->feed['feed_url'])) {
$xml = simplexml_load_string($this->ToUTF8($http->body), 'XML_Element', LIBXML_NOERROR, LIBXML_NOWARNING, LIBXML_NOCDATA);
if ($xml !== false) {
$DB = GetDB();
foreach ($xml->xpath('//videos/video') as $xvideo) {
// Check for duplicates, and skip
if ($DB->QueryCount('SELECT COUNT(*) FROM `tbx_video_feed_history` WHERE `feed_id`=? AND `unique_id`=?', array($this->feed['feed_id'], $xvideo->id->val()))) {
continue;
}
// Setup defaults
$video = $this->defaults;
$video['title'] = $xvideo->title->val();
$video['description'] = $xvideo->description->val();
$video['tags'] = Tags::Format($xvideo->tags->val());
if (empty($video['description'])) {
$video['description'] = $video['title'];
}
// Process <clips>
$clips = array();
$screens = array();
foreach ($xvideo->xpath('./clips/clip') as $xclip) {
$video['duration'] += $xclip->duration;
$clip_url = $xvideo->clip_url->val();
$flv = $xclip->flv->val();
// Account for malformed feeds where the clip_url contains the URL to the video
// file rather than the required root URL
if (strstr($clip_url, $flv) === false) {
$clip_url = $clip_url . $flv;
}
$clips[] = array('type' => 'URL', 'clip' => $clip_url);
foreach ($xclip->xpath('./screens/screen') as $xscreen) {
$screen_url = $xvideo->screen_url->val();
$screen = $xscreen->val();
// Account for malformed feeds where the screen_url contains the URL to the image
// file rather than the required root URL
if (strstr($screen_url, $screen) === false) {
$screen_url = $screen_url . $screen;
}
$screens[] = array('thumbnail' => $screen_url);
}
}
if (count($clips) > 0) {
$best_category = GetBestCategory(join(' ', array($video['title'], $video['description'], $video['tags'])));
if (!empty($best_category)) {
$video['category_id'] = $best_category;
}
if ($this->feed['flag_convert']) {
$video['status'] = STATUS_QUEUED;
$video['next_status'] = $this->feed['status'];
}
$video['video_id'] = DatabaseAdd('tbx_video', $video);
DatabaseAdd('tbx_video_custom', $video);
DatabaseAdd('tbx_video_stat', $video);
if (!$video['is_private']) {
Tags::AddToFrequency($video['tags']);
}
$video['queued'] = time();
if ($this->feed['flag_convert']) {
DatabaseAdd('tbx_conversion_queue', $video);
}
if ($this->feed['flag_thumb']) {
DatabaseAdd('tbx_thumb_queue', $video);
}
UpdateCategoryStats($video['category_id']);
$video_dir = new Video_Dir(Video_Dir::DirNameFromId($video['video_id']));
foreach ($clips as $clip) {
$clip['video_id'] = $video['video_id'];
DatabaseAdd('tbx_video_clip', $clip);
}
$display_thumbnail = null;
foreach ($screens as $screen) {
$thttp = new HTTP();
if ($thttp->Get($screen['thumbnail'], $screen['thumbnail'])) {
$temp_file = $video_dir->AddTempFromVar($thttp->body, JPG_EXTENSION);
$imgsize = @getimagesize($temp_file);
if ($imgsize !== false) {
if (Video_Thumbnail::CanResize()) {
$local_filename = Video_Thumbnail::Resize($temp_file, Config::Get('thumb_size'), Config::Get('thumb_quality'), $video_dir->GetThumbsDir());
} else {
$local_filename = $video_dir->AddThumbFromFile($temp_file, JPG_EXTENSION);
}
$local_filename = str_replace(Config::Get('document_root'), '', $local_filename);
$thumb_id = DatabaseAdd('tbx_video_thumbnail', array('video_id' => $video['video_id'], 'thumbnail' => $local_filename));
if (empty($display_thumbnail)) {
$display_thumbnail = $thumb_id;
}
}
}
}
$video_dir->ClearTemp();
if (!empty($display_thumbnail)) {
$DB->Update('UPDATE `tbx_video` SET `display_thumbnail`=? WHERE `video_id`=?', array($display_thumbnail, $video['video_id']));
}
$DB->Update('INSERT INTO `tbx_video_feed_history` VALUES (?,?)', array($this->feed['feed_id'], $xvideo->id->val()));
$imported++;
}
}
$DB->Update('UPDATE `tbx_video_feed` SET `date_last_read`=? WHERE `feed_id`=?', array(Database_MySQL::Now(), $this->feed['feed_id']));
UpdateSponsorStats($this->feed['sponsor_id']);
}
// Start up the thumbnail and converson queues if needed
if (!Config::Get('flag_using_cron')) {
if ($this->feed['flag_convert']) {
ConversionQueue::Start();
}
if ($this->feed['flag_thumb']) {
ThumbQueue::Start();
}
}
}
return $imported;
}
