function handle($params)
{
$app = Dataface_Application::getInstance();
$query = $app->getQuery();
try {
if (!@$_POST['-table']) {
throw new Exception("No table was specified");
}
$vals = array();
foreach ($query as $k => $v) {
if ($k and $k[0] != '-') {
$vals[$k] = $v;
}
}
$record = new Dataface_Record($_POST['-table'], array());
$record->setValues($vals);
if (!$record->checkPermission('ajax_save')) {
throw new Exception("Permission Denied", 502);
}
$res = $record->save(null, true);
if (PEAR::isError($res)) {
error_log($res->getMessage(), $res->getCode());
throw new Exception("Failed to save record due to a server error. See log for details.");
}
$this->out(array('code' => 200, 'message' => 'Successfully inserted record.', 'recordId' => $record->getId()));
} catch (Exception $ex) {
$this->out(array('code' => $ex->getCode(), 'message' => $ex->getMessage()));
}
}
function handle(&$params)
{
$app =& Dataface_Application::getInstance();
$tt = new Dataface_TranslationTool();
if (!Dataface_Table::tableExists('dataface__translation_submissions', false)) {
$tt->createTranslationSubmissionsTable();
header('Location: ' . $app->url(''));
exit;
}
if (!@$_POST['--submit']) {
df_display(array('query' => $app->getQuery(), 'success' => @$_REQUEST['--success']), 'Dataface_submit_translation.html');
exit;
} else {
if (@$_POST['subject']) {
// This is a dummy field - possible hacking attempt
header('Location: ' . $app->url('-action=list'));
exit;
}
if (@$_POST['--recordid']) {
$record = df_get_record_by_id($_POST['--recordid']);
$values = array('record_id' => @$_POST['--recordid'], 'language' => @$_POST['--language'], 'url' => @$_POST['--url'], 'original_text' => @$_POST['--original_text'], 'translated_text' => @$_POST['--translated_text'], 'translated_by' => @$_POST['--translated_by']);
$trec = new Dataface_Record('dataface__translation_submissions', array());
$trec->setValues($values);
$trec->save();
$email = <<<END
The following translation was submitted to the web site {$app->url('')}:
Translation for record {$record->getTitle()} which can be viewed at {$record->getURL('-action=view')}.
This translation was submitted by {$_POST['--translated_by']} after viewing the content at {$_POST['--url']}.
The original text that was being translated is as follows:
{$_POST['--original_text']}
The translation proposed by this person is as follows:
{$_POST['--translated_text']}
For more details about this translation, please visit {$trec->getURL('-action=view')}.
END;
if (@$app->_conf['admin_email']) {
mail($app->_conf['admin_email'], 'New translation submitted', $email);
}
if (@$_POST['--redirect'] || @$_POST['--url']) {
$url = @$_POST['--redirect'] ? $_POST['--redirect'] : $_POST['--url'];
header('Location: ' . $url . '&--msg=' . urlencode('Thank you for your submission.'));
exit;
} else {
header('Location: ' . $app->url('') . '&--success=1&--msg=' . urlencode('Thank you for your submission.'));
exit;
}
} else {
trigger_error("No record id was provided", E_USER_ERROR);
}
}
}
public function saveResource($url, $content, $contentType)
{
$app = Dataface_Application::getInstance();
$res = new Dataface_Record('job_content', array());
$res->setValues(array('job_id' => $this->job->getRecord()->val('job_id'), 'url' => $url, 'url_hash' => md5($url), 'content_type' => $contentType, 'content' => $content));
$old = $app->_conf['multilingual_content'];
$app->_conf['multilingual_content'] = 0;
$result = $res->save();
$app->_conf['multilingual_content'] = $old;
if (PEAR::isError($res)) {
throw new Exception($res->getMessage());
}
}
function testCrawlSite()
{
$s = new Dataface_Record('websites', array());
$s->setValues(array('website_url' => 'http://solutions.weblite.ca/', 'source_language' => 'en', 'target_language' => 'fr', 'website_name' => 'Site 1 french', 'active' => 1, 'base_path' => '/fr/', 'host' => 'localhost'));
$s->save();
$site = SweteSite::loadSiteById($s->val('website_id'));
$crawler = new SiteCrawler();
$crawler->site = $site;
$crawler->startingPoint = 'http://solutions.weblite.ca/';
$crawler->depth = 3;
$crawler->crawl();
//print_r($crawler->root);
}
function testPage()
{
$url = DATAFACE_SITE_URL . '/tests/testsites/site1/index.html';
$site = new Dataface_Record('websites', array());
$site->setValues(array('website_url' => df_absolute_url(DATAFACE_SITE_URL . '/tests/testsites/site1/'), 'source_language' => 'en', 'target_language' => 'fr', 'website_name' => 'Site 1 French', 'active' => 1, 'base_path' => DATAFACE_SITE_URL . '/proxies/site1/', 'host' => $_SERVER['HTTP_HOST']));
$site->save();
df_q("delete from site_text_filters where website_id='" . addslashes($site->val('website_id')) . "'");
$server = new ProxyServer();
$server->site = SweteSite::loadSiteById($site->val('website_id'));
$server->SERVER = array('REQUEST_METHOD' => 'get');
$server->URL = df_absolute_url(DATAFACE_SITE_URL . '/proxies/site1/index.html');
$server->buffer = true;
$server->handleRequest();
$doc = new DOMDocument();
$doc->loadHtml(file_get_contents('tests/testsites/site1_output/index.out.html'));
$expected = $doc->saveHtml();
//echo $server->contentBuffer;
$doc2 = new DOMDocument();
$doc2->loadHtml($server->contentBuffer);
$actual = $doc2->saveHtml();
//$this->assertEquals(trim($expected), trim($actual));
// Cancelled this test because WTF!!!! Even if I print the actual output, copy it to the file
// and compare it to itself, it still fails!!!! WTF!!!!
}
public function addMessage($content)
{
$note = new Dataface_Record('job_notes', array());
$note->setValues(array('job_id' => $this->getJob()->getRecord()->val('job_id'), 'note_content' => $content, 'date_posted' => date('Y-m-d H:i:s'), 'posted_by' => $this->getUsername()));
$res = $note->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage(), $res->getCode());
}
return $note;
}
function handle($params)
{
if (!defined('DISABLE_reCAPTCHA')) {
define('DISABLE_reCAPTCHA', 1);
}
import('Dataface/QuickForm.php');
Dataface_QuickForm::$TRACK_SUBMIT = false;
$app = Dataface_Application::getInstance();
$query = $app->getQuery();
$errors = null;
try {
if (!@$_POST['-table']) {
throw new Exception("No table specified");
}
$table = $_POST['-table'];
$rec = new Dataface_Record($table, array());
$tableObj = $rec->_table;
$fields = array();
if (!$rec->checkPermission('new')) {
throw new Exception("Failed to insert record. Permission denied");
}
foreach ($_POST as $k => $v) {
if ($k[0] == '-') {
continue;
}
$fields[] = $k;
$rec->setValue($k, $v);
if (!$rec->checkPermission('new', array('field' => $k))) {
throw new Exception(sprintf("Failed to insert record because you do not have permission to insert data into the %s column", $k));
}
}
$form = df_create_new_record_form($table, $fields);
$form->_flagSubmitted = true;
$res = $form->validate();
if (!$res) {
$errors = $form->_errors;
throw new Exception('Validation error', REST_INSERT_VALIDATION_ERROR);
}
$res = $rec->save(null, true);
if (PEAR::isError($res)) {
throw new Exception("Failed to insert record due to a server error: " . $res->getMessage(), 500);
}
$out = array();
$vals = $rec->strvals();
foreach ($vals as $k => $v) {
if ($rec->checkPermission('view')) {
$out[$k] = $v;
}
}
$this->out(array('code' => 200, 'message' => 'Record successfully inserted', 'record' => $out));
exit;
} catch (Exception $ex) {
$this->out(array('code' => $ex->getCode(), 'message' => $ex->getMessage(), 'errors' => $errors));
exit;
}
}
function submitTranslation(&$record, $params = array())
{
if (!Dataface_Table::tableExists('dataface__translation_submissions', false)) {
$this->createTranslationSubmissionsTable();
}
$trec = new Dataface_Record('dataface__translation_submissions', array());
$trec->setValues($params);
$trec->save();
}
public function import()
{
$fh = fopen($this->inputFilePath, 'r');
if (!$fh) {
throw new Exception(sprintf("Failed to open input file '%s'", $this->inputFilePath));
}
$headers = array_flip(fgetcsv($fh, 0, $this->separator));
$required_fields = array('normalized_string', 'normalized_translation_value');
if (!isset($this->targetTranslationMemory)) {
$required_fields[] = 'translation_memory_uuid';
}
foreach ($required_fields as $f) {
if (!array_key_exists($f, $headers)) {
throw new Exception(sprintf("Missing required column heading: %s", $f));
}
}
while (($row = fgetcsv($fh, 0, $this->separator)) !== false) {
$string = $row[$headers['normalized_string']];
$translation = $row[$headers['normalized_translation_value']];
$translationMemory = $this->targetTranslationMemory;
$tmuuid = $row[$headers['translation_memory_uuid']];
if (!isset($translationMemory)) {
$translationMemory = XFTranslationMemory::loadTranslationMemoryByUuid($tmuuid);
}
if (!isset($translationMemory)) {
$this->errors[] = array('row' => $row, 'message' => 'No translation memory assigned.');
$this->failed++;
continue;
}
$strRec = XFTranslationMemory::addString($string, $translationMemory->getSourceLanguage());
$res = df_q(sprintf("select string_id from translation_miss_log where string_id=%d and translation_memory_id=%d", $strRec->val('string_id'), $translationMemory->getRecord()->val('translation_memory_id')));
if (mysql_num_rows($res) == 0) {
@mysql_free_result($res);
// This string is not in the translation miss log yet. We
// will import it now
$tlogEntry = new Dataface_Record('translation_miss_log', array());
$nstr = TMTools::normalize($string);
$trimStripped = trim(strip_tags($nstr));
if (!$trimStripped) {
continue;
}
if (preg_match('/^[0-9 \\.,%\\$#@\\(\\)\\!\\?\'":\\+=\\-\\/><]*$/', $trimStripped)) {
continue;
}
// If the string is just a number or non-word we just skip it.
//$estr = TMTools::normalize(TMTools::encode($nstr, $junk));
// We don't need to encode the string
$res = df_q(sprintf("select website_id from websites where translation_memory_id=%d", $translationMemory->getRecord()->val('translation_memory_id')));
if (!$res) {
$this->failed++;
$this->errors[] = array('row' => $row, 'message' => sprintf("No website found for translation memory %d", $translationMemory->getRecord()->val('translation_memory_id')));
continue;
}
list($websiteId) = mysql_fetch_row($res);
@mysql_free_result($res);
$hstr = md5($string);
$tlogEntry->setValues(array('http_request_log_id' => null, 'string' => $string, 'normalized_string' => $string, 'encoded_string' => $string, 'string_hash' => $hstr, 'date_inserted' => date('Y-m-d H:i:s'), 'website_id' => $websiteId, 'source_language' => $translationMemory->getSourceLanguage(), 'destination_language' => $translationMemory->getDestinationLanguage(), 'translation_memory_id' => $translationMemory->getRecord()->val('translation_memory_id'), 'string_id' => $strRec->val('string_id')));
$res = $tlogEntry->save();
if (PEAR::isError($res)) {
$this->errors[] = array('row' => $row, 'message' => 'Failed to insert translation miss log entry: ' . $res->getMessage());
}
}
if (@trim($translation)) {
try {
$translationMemory->setTranslationStatus($string, $translation, XFTranslationMemory::TRANSLATION_APPROVED);
} catch (Exception $ex) {
$this->failed++;
$this->errors[] = array('row' => $row, 'message' => 'Failed to set translation status: ' . $ex->getMessage());
continue;
}
} else {
// No translation provided we don't need to import the translation
}
$this->succeeded++;
}
}
function field__properties($record)
{
$p = df_get_record('webpage_properties', array('webpage_id' => '=' . $record->val('webpage_id')));
if (!$p) {
$p = new Dataface_Record('webpage_properties', array());
$p->setValue('webpage_id', $record->val('webpage_id'));
$p->pouch['webpage'] = $record;
$p->save();
}
return $p;
}
/**
* @brief Creates a new webpage with the given source path.
*
* @see newWebpageWithProxifiedUrl() for details on other arguments.
*/
public function newWebpageWithSrcPath($vals, $checkExists = true, $secure = false)
{
if (!@$vals['webpage_url']) {
throw new Exception("No path provided");
}
if ($checkExists) {
$page = $this->loadWebpageBySrcPath($vals['webpage_url']);
if ($page) {
throw new Exception("Webpage already exists");
}
}
$vals['webpage_url'] = SweteTools::normalizeUrl($vals['webpage_url']);
if (!isset($vals['webpage_content'])) {
$vals['webpage_content'] = '';
}
$rec = new Dataface_Record('webpages', array());
$rec->setValues($vals);
$res = $rec->save($this->getSourceLanguage(), $secure);
if (PEAR::isError($res)) {
throw new Exception($res->getMessage(), $res->getCode());
}
$page = $this->loadWebpageBySrcPath($vals['webpage_url']);
if (!$page) {
throw new Exception("We have inserted a page, but cannot seem to load it.");
}
return $page;
}
/**
* @brief Sets the status of the current page to the specified approval status. This will add
* a record to the webpage_versions table with the specified status id. I.e. this effectively
* takes a snapshot of the record.
*
* @param int $status_id The status id to set. This should be one of:
* <ul>
* <li>SweteWebpage::STATUS_NEW</li>
* <li>SweteWebpage::STATUS_CHANGED</li>
* <li>SweteWebpage::STATUS_PENDING_APPROVAL</li>
* <li>SweteWebpage::STATUS_APPROVED</li>
* </ul>
* @param string $username The username of the user who is to be attributed with posting this
* update.
* @param string $comments Comments associated with this status change. This can be used
* to add a reason for the status change.
* @param boolean $secure Whether to make this subject to standard Xataface security.
* @returns Dataface_Record The resulting webpage_versions record.
*/
public function setStatus($status_id, $username, $comments, $secure = false)
{
$rec = new Dataface_Record('webpage_versions', array());
//$rec->setValues($this->_rec->vals());
$rec->setValues(array('approval_status' => $status_id, 'language' => $this->getLanguage(), 'posted_by' => $username, 'comments' => $comments, 'webpage_id' => $this->_rec->val("webpage_id"), 'page_content' => $this->_rec->val('webpage_content'), 'date_posted' => date('Y-m-d H:i:s')));
$res = $rec->save($secure);
if (PEAR::isError($res)) {
throw new Exception(mysql_error(df_db()));
}
return $rec;
}
/**
* @brief Handles an HTTP request. Processes the inputs and returns the
* correct output.
*/
public function handleRequest()
{
$this->mark("handleRequest: " . $this->URL);
$url = $this->URL;
$siteId = $this->site->getRecord()->val('website_id');
if (file_exists('sites/' . basename($siteId) . '/Delegate.php')) {
require_once 'sites/' . basename($siteId) . '/Delegate.php';
$clazz = 'sites_' . intval($siteId) . '_Delegate';
if (class_exists($clazz) and method_exists($clazz, 'init')) {
call_user_func(array($clazz, 'init'));
}
}
$proxyWriter = $this->site->getProxyWriter();
$proxyWriter->useHtml5Parser = $this->useHtml5Parser;
$logger = $this->logger;
$logger->proxyRequestUrl = $url;
$isPost = strtolower($this->SERVER['REQUEST_METHOD']) === 'post';
if ($isPost) {
// We cannot cache post requests.
// The cacher knows this, but let's make doubly sure.
Dataface_Application::getInstance()->_conf['nocache'] = 1;
}
if (!$isPost and !preg_match('#\\.(ico|ICO|gif|GIF|jpg|JPG|jpeg|JPEG|SWF|swf|css|CSS|png|PNG|pdf|PDF|doc|DOC|svg|SVG|fla|FLA|zip|ZIP|js|JS)$#', $url)) {
$logger->proxyRequestHeaders = serialize(apache_request_headers());
$logger->proxyRequestPostVars = serialize($this->POST);
$logger->proxyRequestMethod = $this->SERVER['REQUEST_METHOD'];
$logger->settingsSiteId = $this->site->getRecord()->val('settings_site_id');
$this->mark('Loading webpage');
$page = $this->site->loadWebpageByProxifiedUrl($url);
$this->mark('Webpage loaded');
if ($page) {
$logger->webpageId = $page->getRecord()->val('webpage_id');
if (!$page->getRecord()->val('active')) {
$logger->webpageNotUsedReason = 'Not active';
} else {
$this->mark('Loading latest version with status ' . $this->pageStatus);
$version = $page->getLastVersionWithStatus($this->pageStatus, $this->site->getDestinationLanguage());
$this->mark('Version loaded');
if ($version) {
$logger->webpageVersionId = $version->val('webpage_version_id');
//print_r($version->vals());exit;
$this->mark('Proxifying html');
$out = $this->site->getProxyWriter()->proxifyHtml($version->val('page_content'));
$this->mark('Finished proxifying html');
$logger->outputContent = $out;
$this->header('Content-Length: ' . strlen($out));
$this->header('Connection: close');
$this->header('Cache-Control: max-age=36000');
$this->header('X-SWeTE-Handler: ProxyServer/Static page/v' . $logger->webpageVersionId . '/' . __LINE__);
$this->output($out);
while (@ob_end_flush()) {
}
flush();
$this->mark('Flushed contents to browser');
$logger->outputResponseHeaders = serialize(headers_list());
$logger->save();
return;
}
}
}
}
//Wasn't found so we try to load the
$this->mark('Getting the source page');
if (@$this->inputContent) {
// The content was provided.. we don't need to try to load it from
// the source site
$client = $this->createClientForInputContent();
} else {
if ($this->liveCache and $this->liveCache->client) {
$client = $this->liveCache->client;
} else {
$client = $this->getSourcePage();
}
}
$this->mark('Source page loaded');
//echo "We got the source page.";
//print_r($client->headers);
$isHtml = preg_match('/html|xml/', $client->contentType);
$isJson = (preg_match('/json/', $client->contentType) or $client->content and ($client->content[0] == '{' or $client->content[0] == '['));
$isCSS = preg_match('/css/', $client->contentType);
$json = null;
if ($isJson) {
$json = json_decode($client->content, true);
if (isset($json)) {
$html = $proxyWriter->jsonToHtml($json);
$isHtml = isset($html);
if ($isHtml) {
$client->content = $html;
} else {
$isJson = false;
}
} else {
$isJson = false;
}
}
$delegate = new ProxyClientPreprocessor($this->site->getRecord()->val('website_id'));
$delegate->preprocessHeaders($client->headers);
$headers = $proxyWriter->proxifyHeaders($client->headers, true);
$locHeaders = preg_grep('/^Location:/i', $headers);
// Let's see if this should be a passthru
$translationMode = $delegate->getTranslationMode($client);
if (!$isHtml and !$isCSS and $translationMode !== ProxyClient::TRANSLATION_MODE_TRANSLATE) {
//$skip_decoration_phase = true;
$cacheControlSet = false;
foreach ($headers as $h) {
if (preg_match('/^Cache-control:(.*)$/i', $h, $matches)) {
// We need to respect caching rules.
// If this content is private then we cannot cache it
$cacheControlSet = true;
if (preg_match('/private|no-store|max-age=0|s-maxage=0/', $matches[1])) {
Dataface_Application::getInstance()->_conf['nocache'];
}
}
$this->header($h);
}
$this->header('Content-Length: ' . strlen($client->content));
$this->header('Connection: close');
$this->header('X-SWeTE-Handler: ProxyServer Unprocessed/Non-HTML/Non-CSS/' . __LINE__);
//if ( !$cacheControlSet ) $this->header('Cache-Control: max-age=3600, public');
$this->output($client->content);
if (!$this->buffer) {
while (@ob_end_flush()) {
}
flush();
}
$this->mark('Flushed non-html content');
return;
}
$stats = array();
if ($isHtml and $translationMode !== ProxyClient::TRANSLATION_MODE_NOTRANSLATE and !$locHeaders) {
$this->mark('Preprocessing page content');
$client->content = $delegate->preprocess($client->content);
$this->mark('Finished preprocessing');
$logger->requestDate = date('Y-m-d H:i:s');
$logger->proxyRequestUrl = $url;
$logger->proxyRequestHeaders = serialize(apache_request_headers());
$logger->proxyRequestPostVars = serialize($this->POST);
$logger->proxyRequestMethod = $this->SERVER['REQUEST_METHOD'];
$logger->websiteId = $this->site->getRecord()->val('website_id');
$this->mark('Getting the profile for this page');
$profile = $this->site->getProfile($proxyWriter->stripBasePath($url));
$this->mark('Profile retrieved');
if ($profile and $profile->val('enable_live_translation')) {
if (isset($this->liveCache)) {
$this->liveCache->live = true;
}
try {
$translation_memory_id = $profile->val('translation_memory_id');
if (!$translation_memory_id) {
throw new Exception("No translation memory is set up for this page: " . $url);
}
if (isset($this->liveCache)) {
$this->liveCache->translationMemoryId = $translation_memory_id;
}
require_once 'modules/tm/lib/XFTranslationMemory.php';
//$minApprovalLevel = $profile->val('live_translation_min_approval_level');
$minApprovalLevel = 3;
//XFTranslationMemory::TRANSLATION_APPROVED;
$logger->liveTranslationEnabled = 1;
$logger->liveTranslationMinStatus = $minApprovalLevel;
$this->mark('Loading translation memory: ' . $translation_memory_id);
$tm = XFTranslationMemory::loadTranslationMemoryById($translation_memory_id);
$this->mark('Translation memory loaded');
$logger->translationMemoryId = $translation_memory_id;
$proxyWriter->setTranslationMemory($tm);
$proxyWriter->setMinTranslationStatus($minApprovalLevel);
$this->mark('Translating html');
$client->content = $proxyWriter->translateHtml($client->content, $stats, $this->logTranslationMisses);
$this->mark('Translation complete');
//print_r($stats);exit;
$logger->liveTranslationHits = $stats['matches'];
$logger->liveTranslationMisses = $stats['misses'];
} catch (Exception $ex) {
error_log($ex->getMessage());
}
}
$this->mark('PROXIFY HTML START');
$client->content = $proxyWriter->proxifyHtml($client->content);
$this->mark('PROXIFY HTML END');
//$client->content = preg_replace('#</head>#', '<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script><script src="http://localhost/sfutheme/js/newclf.js"></script><link rel="stylesheet" type="text/css" href="http://localhost/sfutheme/css/newclf.css"/></head>', $client->content);
if ($isJson) {
$client->content = $proxyWriter->htmlToJson($json, $client->content);
}
} else {
if ($isCSS) {
if (isset($this->liveCache)) {
$this->liveCache->live = true;
}
$logger->requestLoggingEnabled = false;
$this->mark('PROXIFY CSS START');
$client->content = $proxyWriter->proxifyCss($client->content);
$this->mark('PROXIFY CSS END');
} else {
$logger->requestLoggingEnabled = false;
}
}
$cacheControlSet = false;
foreach ($headers as $h) {
if (preg_match('/^Cache-control:(.*)$/i', $h, $matches)) {
// We need to respect caching rules.
// If this content is private then we cannot cache it
$cacheControlSet = true;
if (preg_match('/private|no-store|max-age=0|s-maxage=0/', $matches[1])) {
Dataface_Application::getInstance()->_conf['nocache'] = 1;
}
}
//error_log("Setting header: $h");
$this->header($h);
}
$this->header('Content-Length: ' . strlen($client->content));
$this->header('X-SWeTE-Handler: ProxyServer Live/Processed/' . __LINE__);
$this->header('Connection: close');
// We won't add our own cache-control. We'll let the source site decide this and send
// their own headers.
if (!$cacheControlSet and class_exists('Xataface_Scaler') and !@Dataface_Application::getInstance()->_conf['nocache']) {
//$this->header('Cache-Control: max-age=3600');
}
if ($this->inputContentType === 'text/plain') {
$client->content = trim(strip_tags($client->content));
}
$this->output($client->content);
if (!$this->buffer) {
while (@ob_end_flush()) {
}
flush();
if (isset($this->liveCache)) {
$this->mark('The live cache is enabled. Lets set the content');
$this->liveCache->siteId = $this->site->getRecord()->val('website_id');
$this->liveCache->sourceLanguage = $this->site->getSourceLanguage();
$this->liveCache->proxyLanguage = $this->site->getDestinationLanguage();
$this->liveCache->proxyUrl = $this->site->getProxyUrl();
$this->liveCache->siteUrl = $this->site->getSiteUrl();
$this->liveCache->sourceDateLocale = $this->site->getRecord()->val('source_date_locale');
$this->liveCache->targetDateLocale = $this->site->getRecord()->val('target_date_locale');
if ($this->site->getRecord()->val('translation_parser_version')) {
$this->liveCache->translationParserVersion = intval($this->site->getRecord()->val('translation_parser_version'));
}
$this->liveCache->content = $client->content;
$this->liveCache->headers = headers_list();
$this->liveCache->calculateExpires();
if ($this->logTranslationMisses) {
$this->liveCache->skipLiveCache = true;
} else {
$this->liveCache->skipLiveCache = false;
}
$this->mark('About to check if resource can be cached.');
if ($this->liveCache->expires > time()) {
$this->mark('Caching resource for live cache');
$this->liveCache->save();
if (!$this->liveCache->noServerCache) {
$this->liveCache->saveContent();
}
$this->mark('Finished cashing resource for live cache.');
} else {
if ($this->enableProfiling) {
$this->mark('Resource cannot be cached with live cache. Expiry is ' . date($this->liveCache->expires) . ' but now is ' . time() . '.');
}
$this->mark('Saving just the cache info entry');
$this->liveCache->save();
}
}
}
$this->mark('Content flushed');
$logger->outputContent = $client->content;
$logger->outputResponseHeaders = serialize(headers_list());
$logger->save();
$this->mark('Loading the translation miss log');
$tlogEntry = new Dataface_Record('translation_miss_log', array());
if ($this->logTranslationMisses and @$stats['log']) {
$this->mark('ITERATING TRANSLATION MISSES START (' . count($stats['log']) . ')');
foreach ($stats['log'] as $str) {
$tlogEntry = new Dataface_Record('translation_miss_log', array());
$nstr = TMTools::normalize($str);
$trimStripped = trim(strip_tags($nstr));
if (!$trimStripped) {
continue;
}
if (preg_match('/^[0-9 \\.,%\\$#@\\(\\)\\!\\?\'":\\+=\\-\\/><]*$/', $trimStripped)) {
continue;
}
// If the string is just a number or non-word we just skip it.
$estr = TMTools::normalize(TMTools::encode($nstr, $junk));
$strRec = XFTranslationMemory::addString($estr, $this->site->getSourceLanguage());
$hstr = md5($estr);
$tlogEntry->setValues(array('http_request_log_id' => $logger->getRecord()->val('http_request_log_id'), 'string' => $str, 'normalized_string' => $nstr, 'encoded_string' => $estr, 'string_hash' => $hstr, 'date_inserted' => date('Y-m-d H:i:s'), 'website_id' => $this->site->getRecord()->val('website_id'), 'source_language' => $this->site->getSourceLanguage(), 'destination_language' => $this->site->getDestinationLanguage(), 'translation_memory_id' => @$translation_memory_id, 'string_id' => $strRec->val('string_id')));
$res = $tlogEntry->save();
if (PEAR::isError($res)) {
//throw new Exception($res->getMessage());
// This will throw an error if there is a duplicate... we don't care... we're not interested in duplicates
}
}
$this->mark('ITERATING TRANSLATION MISSES END');
}
return;
}
function testCompileStaticSite()
{
$siteRec = $this->staticSite;
$job = SweteJob::createJob($this->staticSite);
$this->assertFalse($job->getRecord()->val('compiled'), 'Job is not compiled yet but has compiled flag set.');
//add webpage to static site
$webpageContent = 'Test string 1';
$pg = new Dataface_Record('webpages', array());
$pg->lang = 'en';
$pg->setValues(array('website_id' => $this->staticSite->getRecord()->val('website_id'), 'webpage_url' => 'index', 'webpage_content' => $webpageContent, 'active' => 1, 'posted_by' => 'test_user'));
$res = $pg->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage());
}
$pgid = $pg->val('webpage_id');
$page1 = SweteWebpage::loadByUrl($this->staticSite->getRecord()->val('website_id'), 'index', 'en');
$this->assertTrue($page1 instanceof SweteWebpage);
//add webpage to job
$job->addWebpage($page1);
$this->assertTrue($job->containsWebpage($page1));
$translatable = df_get_record('job_translatable', array('job_id' => '=' . $job->getRecord()->val('job_id')));
$this->assertFalse($translatable);
$job->compile();
$res = df_q("select count(*) from job_translatable where job_id='" . addslashes($job->getRecord()->val('job_id')) . "'");
list($num) = mysql_fetch_row($res);
$this->assertEquals(1, $num, 'Should be one translatable created for job.');
$translatable = df_get_record('job_translatable', array('job_id' => '=' . $job->getRecord()->val('job_id')));
$this->assertTrue($translatable instanceof Dataface_Record, 'No translatable found for job.');
$this->assertEquals($webpageContent, strip_tags($translatable->val('full_contents')));
$job->refresh();
$this->assertEquals(1, $job->getRecord()->val('compiled'), 'Job compiled flag is not set but should be');
}
function handle($params)
{
$app = Dataface_Application::getInstance();
if (!$_POST) {
throw new Exception("Commit requires post");
}
$data = @$_POST['--data'];
if (!$data) {
throw new Exception("No data provided");
}
$data = json_decode($data, true);
$updates = array();
$inserts = array();
$deletes = array();
if (isset($data['inserts'])) {
$inserts =& $data['inserts'];
}
if (isset($data['updates'])) {
$updates =& $data['updates'];
}
if (isset($data['deletes'])) {
$deletes =& $data['deletes'];
}
$numFailures = 0;
$numSuccesses = 0;
$deleteResponses = array();
// Let's do the deletes first
foreach ($deletes as $deleteInfo) {
$response = array();
$deleteResponses[] =& $response;
$record = df_get_record_by_id($deleteInfo['id']);
if (!$record) {
$response['message'] = 'Record ' . $deleteInfo['id'] . ' could not be found.';
$response['code'] = 404;
$numFailures++;
} else {
$res = $response->delete(true);
if (PEAR::isError($res)) {
$response['message'] = $res->getMessage();
$response['code'] = $res->getCode();
$numFailures++;
} else {
$response['message'] = 'Deleted record ' . $deleteInfo['id'] . '.';
$response['code'] = 200;
$response['recordId'] = $deleteInfo['id'];
$numSuccesses++;
}
}
}
$insertResponses = array();
foreach ($inserts as $insertInfo) {
$response = array();
$insertResponses[] =& $response;
$record = new Dataface_Record($insertInfo['table'], array());
$record->setValues($insertInfo['data']);
$res = $record->save(null, true);
if (PEAR::isError($res)) {
$response['message'] = $res->getMessage();
$response['code'] = $res->getCode();
$numFailures++;
} else {
$response['message'] = 'Inserted record';
$response['code'] = $res->getCode();
$response['recordId'] = $record->getId();
$response['version'] = $record->getVersion();
$numSuccesses++;
}
}
$updateResponses = array();
foreach ($updates as $updateInfo) {
$response = array();
$insertResponses[] =& $response;
$record = df_get_record_by_id($updateInfo['id']);
if (!$record) {
$response['message'] = 'Record ' . $updateInfo['id'] . ' could not be found.';
$response['code'] = 404;
$numFailures++;
} else {
$record->setValues($updateInfo['data']);
$res = $record->save(null, true);
if (PEAR::isError($res)) {
$response['message'] = $res->getMessage();
$response['code'] = $res->getCode();
$numFailures++;
} else {
$response['message'] = 'Updated record';
$response['code'] = 200;
$response['recordId'] = $record->getId();
$response['version'] = $record->getVersion();
$numSuccesses++;
}
}
}
header('Content-type: text/json; charset="' . $app->_conf['oe'] . '"');
$out = array('code' => ($numFailures == 0 and $numSuccesses > 0) ? 200 : $numSuccesses > 0 ? 201 : 202, 'message' => $numSuccesses . ' successes. ' . $numFailures . ' failures.', 'numSuccesses' => $numSuccesses, 'numFailures' => $numFailures, 'responses' => array('updates' => $updateResponses, 'inserts' => $insertResponses, 'deletes' => $deleteResponses));
echo json_encode($out);
}
function handle(&$params)
{
$app = Dataface_Application::getInstance();
if (!isset($_GET['code'])) {
// We need this parameter or we can do nothing.
return PEAR::raiseError(df_translate('actions.activate.MESSAGE_MISSING_CODE_PARAMETER', 'The code parameter is missing from your request. Validation cannot take place. Please check your url and try again.'), DATAFACE_E_ERROR);
}
// Step 0: Find out what the redirect URL will be
// We accept --redirect markers to specify which page to redirect
// to after we're done. This will usually be the page that the
// user was on before they went to the login page.
if (isset($_SESSION['--redirect'])) {
$url = $_SESSION['--redirect'];
} else {
if (isset($_SESSION['-redirect'])) {
$url = $_SESSION['-redirect'];
} else {
if (isset($_REQUEST['--redirect'])) {
$url = $_REQUEST['--redirect'];
} else {
if (isset($_REQUEST['-redirect'])) {
$url = $_REQUEST['-redirect'];
} else {
$url = $app->url('-action=' . $app->_conf['default_action']);
}
}
}
}
if (strpos($url, '?') === false) {
$url .= '?';
}
// Step 1: Delete all registrations older than time limit
$time_limit = 24 * 60 * 60;
// 1 day
if (isset($params['time_limit'])) {
$time_limit = intval($params['time_limit']);
}
$res = xf_db_query("delete from dataface__registrations \n\t\t\t\twhere registration_date < '" . addslashes(date('Y-m-d H:i:s', time() - $time_limit)) . "'", df_db());
if (!$res) {
error_log(xf_db_error(df_db()));
throw new Exception("Failed to delete registrations due to an SQL error. See error log for details.", E_USER_ERROR);
}
// Step 2: Load the specified registration information
$res = xf_db_query("select registration_data from dataface__registrations\n\t\t\t\twhere registration_code = '" . addslashes($_GET['code']) . "'", df_db());
if (!$res) {
error_log(xf_db_error(df_db()));
throw new Exception("Failed to load registration information due to an SQL error. See error log for details.", E_USER_ERROR);
}
if (xf_db_num_rows($res) == 0) {
// We didn't find any records matching the prescribed code, so
// we redirect the user to their desired page and inform them
// that the registration didn't work.
$msg = df_translate('actions.activate.MESSAGE_REGISTRATION_NOT_FOUND', 'No registration information could be found to match this code. Please try registering again.');
$app->redirect($url . '&--msg=' . urlencode($msg));
}
// Step 3: Check to make sure that there are no other users with the
// same name.
list($raw_data) = xf_db_fetch_row($res);
$values = unserialize($raw_data);
$appdel = $app->getDelegate();
if (isset($appdel) and method_exists($appdel, 'validateRegistrationForm')) {
$res = $appdel->validateRegistrationForm($values);
if (PEAR::isError($res)) {
$msg = $res->getMessage();
$app->redirect($url . '&--msg=' . urlencode($msg));
}
} else {
$res = xf_db_query("select count(*) from \n\t\t\t\t`" . str_replace('`', '', $app->_conf['_auth']['users_table']) . "` \n\t\t\t\twhere `" . str_replace('`', '', $app->_conf['_auth']['username_column']) . "` = '" . addslashes($values[$app->_conf['_auth']['username_column']]) . "'\n\t\t\t\t", df_db());
if (!$res) {
error_log(xf_db_error(df_db()));
throw new Exception("Failed to find user records due to an SQL error. See error log for details.", E_USER_ERROR);
}
list($num) = xf_db_fetch_row($res);
if ($num > 0) {
$msg = df_translate('actions.activate.MESSAGE_DUPLICATE_USER', 'Registration failed because a user already exists by that name. Try registering again with a different name.');
$app->redirect($url . '&--msg=' . urlencode($msg));
}
}
// Step 4: Save the registration data and log the user in.
$record = new Dataface_Record($app->_conf['_auth']['users_table'], array());
$record->setValues($values);
$res = $record->save();
if (PEAR::isError($res)) {
$app->redirect($url . '&--msg=' . urlencode($res->getMessage()));
} else {
$res = xf_db_query("delete from dataface__registrations\n\t\t\t\t\twhere registration_code = '" . addslashes($_GET['code']) . "'", df_db());
if (!$res) {
error_log(xf_db_error(df_db()));
throw new Exception("Failed to clean up old registrations due to an SQL error. See error log for details.", E_USER_ERROR);
}
$msg = df_translate('actions.activate.MESSAGE_REGISTRATION_COMPLETE', 'Registration complete. You are now logged in.');
$_SESSION['UserName'] = $record->strval($app->_conf['_auth']['username_column']);
import('Dataface/Utilities.php');
Dataface_Utilities::fireEvent('after_action_activate', array('record' => $record));
$app->redirect($url . '&--msg=' . urlencode($msg));
}
}
/**
* @brief Imports a node and all of its children into the database.
*
* @param stdClass $node The root node to import.
* @param Dataface_Record $parentPage A record of the @e webpages table
* that represents the parent page of the current page.
*
* @see SiteCrawler for more information about nodes and the properties
* they can have.
*/
public function importNode(stdClass $node, $parentPage = null)
{
$page = null;
if (isset($node->path) and isset($node->httpStatus)) {
$page = df_get_record('webpages', array('webpage_url' => '=' . $node->path, 'website_id' => '=' . $this->site->getRecord()->val('website_id')));
if (!$page) {
$page = new Dataface_Record('webpages', array());
$page->setValues(array('website_id' => $this->site->getRecord()->val('website_id'), 'webpage_url' => $node->path, 'active' => -1, 'posted_by' => $this->username, 'parent_id' => $parentPage ? $parentPage->val('webpage_id') : null, 'is_loaded' => 0));
$res = $page->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage());
}
$this->pagesAdded[] = $page;
} else {
$this->pagesUpdated[] = $page;
}
$page->setValues(array('last_checked' => date('Y-m-d H:i:s'), 'last_checked_response_code' => $node->httpStatus, 'last_checked_content_type' => $node->contentType, 'last_checked_by' => $this->username));
$updateRefreshLog = false;
$translationStats = array();
//if ( $node->content and $this->loadContent and (!$page->val('locked') or $this->overrideLocks) ){
if (@$node->content and @$this->loadContent) {
if ($page->val('locked') and !$this->overrideLocks) {
error_log("Skipping refresh of " . @$node->path . " because the page is locked.");
} else {
$page->setValues(array('last_refresh' => date('Y-m-d H:i:s'), 'last_refresh_response_code' => $node->httpStatus, 'is_loaded' => 1, 'webpage_content' => $node->content));
// Now log the check
$logEntry = new Dataface_Record('webpage_refresh_log', array());
$logEntry->setValues(array('webpage_id' => $page->val('webpage_id'), 'date_checked' => date('Y-m-d H:i:s'), 'response_code' => $node->httpStatus, 'content_type' => $node->contentType, 'content' => $node->content, 'checked_by' => $this->username));
$res = $logEntry->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage());
}
if ($this->translate) {
$pageWrapper = new SweteWebpage($page);
$tmid = $pageWrapper->getTranslationMemoryId(true);
if ($tmid) {
$tm = $this->getTranslationMemory($tmid);
if ($tm) {
import('inc/PageProcessor.php');
$processor = new PageProcessor();
$processor->webpageRefreshLogId = $logEntry->val('refresh_log_id');
$processor->site = $this->site;
$processor->translationMemory = $tm;
$processor->page = $pageWrapper;
$processor->translateMinStatus = $this->translateMinStatus;
$processor->translateMaxStatus = $this->translateMaxStatus;
$processor->logTranslationMisses = $this->logTranslationMisses;
$processor->savePage = false;
$processor->saveTranslationLogRecord = true;
$processor->process();
}
}
}
}
}
$res = $page->save();
// Now log the check
$logEntry = new Dataface_Record('webpage_check_log', array());
$logEntry->setValues(array('webpage_id' => $page->val('webpage_id'), 'date_checked' => date('Y-m-d H:i:s'), 'response_code' => $node->httpStatus, 'content_type' => $node->contentType, 'checked_by' => $this->username));
$res = $logEntry->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage());
}
}
if (isset($node->children) and is_array($node->children)) {
foreach ($node->children as $child) {
$this->importNode($child, $page);
}
}
}
public function process()
{
$this->translationStats = null;
$this->translationMissLogRecord = null;
$this->translatedPage = null;
$proxyWriter = $this->site->getProxyWriter();
$pageWrapper = $this->page;
$page = $pageWrapper->getRecord();
$tmid = null;
if (!isset($this->translationMemory)) {
$tmid = $pageWrapper->getTranslationMemoryId(true);
} else {
$tmid = $this->translationMemory->getRecord()->val('translation_memory_id');
}
$translatedContent = null;
$untranslatedContent = $page->val('webpage_content');
if (!trim($untranslatedContent)) {
// There is nothing to process on this page.
return;
}
if ($tmid or $this->translationMemory) {
if ($this->translationMemory) {
$tm = $this->translationMemory;
} else {
$tm = $this->getTranslationMemory($tmid);
}
if ($tm) {
if ($tm->getSourceLanguage() != $pageWrapper->getLanguage()) {
throw new Exception("Translation memory language does not match the record language. Translation memory source language is " . $tm->getSourceLanguage() . " but the page language is " . $pageWrapper->getLanguage() . '.');
}
$proxyWriter->setTranslationMemory($tm);
$proxyWriter->setMinTranslationStatus($this->translateMinStatus);
$translatedContent = $proxyWriter->translateHtml($untranslatedContent, $translationStats, $this->logTranslationMisses);
$this->translationStats = $translationStats;
$page->setValues(array('last_translation_memory_applied' => date('Y-m-d H:i:s'), 'last_translation_memory_misses' => $translationStats['misses'], 'last_translation_memory_hits' => $translationStats['matches']));
// Let's record the strings in this page.
$res = df_q("delete from webpage_strings where webpage_id='" . addslashes($page->val('webpage_id')) . "'");
if ($proxyWriter->lastStrings) {
//print_r($proxyWriter->lastStrings);exit;
$sqlpre = "insert into webpage_strings (webpage_id,string_id) values ";
$sql = array();
$wpid = $page->val('webpage_id');
foreach ($proxyWriter->lastStrings as $str) {
if (!trim($str)) {
continue;
}
if (preg_match('/^[^\\w]+$/', trim($str))) {
// This is to skip any strings that contain only
// non-word characters(e.g. numbers)
continue;
}
$encStr = TMTools::encode($str, $params);
$strRec = XFTranslationMemory::addString($encStr, $tm->getSourceLanguage());
$sql[] = '(' . $wpid . ',' . $strRec->val('string_id') . ')';
}
$sql = $sqlpre . implode(',', $sql);
df_q($sql);
}
$translatedPage = SweteWebpage::loadById($page->val('webpage_id'), $this->site->getDestinationLanguage());
$translatedPage->getRecord()->setValue('webpage_content', $translatedContent);
$res = $translatedPage->getRecord()->save();
if (PEAR::isError($res)) {
throw new Exception(mysql_error(df_db()));
}
$lastApproved = $translatedPage->getLastVersionWithStatus(SweteWebpage::STATUS_APPROVED);
if ($lastApproved and $lastApproved->val('webpage_content') == $translatedContent) {
$page->setValue('webpage_status', SweteWebpage::STATUS_APPROVED);
} else {
if ($translationStats['matches'] > 0 and $translationStats['misses'] == 0) {
// We have perfect matches in what we are supposed to be translating
// We are either approving this page or we are marking it pending approval
if ($translatedPage->getAutoApprove(true)) {
$page->setValue('webpage_status', SweteWebpage::STATUS_APPROVED);
$lastApproved = $translatedPage->setStatus(SweteWebpage::STATUS_APPROVED);
} else {
$page->setValue('webpage_status', SweteWebpage::STATUS_PENDING_APPROVAL);
}
} else {
if ($translationStats['misses'] > 0) {
$page->setValue('webpage_status', SweteWebpage::STATUS_CHANGED);
} else {
$page->setValue('webpage_status', null);
}
}
}
if ($this->logTranslationMisses and @$translationStats['log']) {
//print_r($translationStats);exit;
foreach ($translationStats['log'] as $str) {
$tlogEntry = new Dataface_Record('translation_miss_log', array());
$nstr = TMTools::normalize($str);
$estr = TMTools::encode($str, $junk);
$hstr = md5($estr);
$strRec = XFTranslationMemory::findString($estr, $this->site->getSourceLanguage());
if (!$strRec) {
$strRec = XFTranslationMemory::addString($estr, $this->site->getSourceLanguage());
}
$tlogEntry->setValues(array('string' => $str, 'normalized_string' => $nstr, 'encoded_string' => $estr, 'string_hash' => $hstr, 'date_inserted' => date('Y-m-d H:i:s'), 'webpage_id' => $page->val('webpage_id'), 'website_id' => $page->val('website_id'), 'source_language' => $this->site->getSourceLanguage(), 'destination_language' => $this->site->getDestinationLanguage(), 'translation_memory_id' => $tmid, 'string_id' => $strRec->val("string_id")));
if (isset($this->webpageRefreshLogId)) {
$tlogEntry->setValue('webpage_refresh_log_id', $this->webpageRefreshLogId);
}
if ($this->saveTranslationLogRecord) {
$res = $tlogEntry->save();
if (PEAR::isError($res)) {
//throw new Exception($res->getMessage());
// This will throw an error if there is a duplicate... we don't care... we're not interested in duplicates
}
}
$this->translationMissLogRecord = $tlogEntry;
}
}
if ($this->savePage) {
$res = $page->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage());
}
}
}
}
}
/**
* Compiles the job inputs into its final form so that it can be worked on. Before
* this step the job just has a loose set of input webpages strings and translation
* misses. This will grab all of the resources that it needs to be able to
* present the job to a translator. This includes loading all resources for all
* pages used into the data structure so that the job doesn't depend on outside
* factors.
*/
public function compile()
{
require_once 'inc/SweteJobPageSucker.php';
try {
$res = SweteDb::q("select tml.webpage_id, tml.translation_miss_log_id, tml.string\n\t\t\t\tfrom \n\t\t\t\t\ttranslation_miss_log tml \n\t\t\t\t\tinner join job_inputs_translation_misses jitm on jitm.translation_miss_log_id=tml.translation_miss_log_id\n\t\t\t\t\twhere jitm.job_id='" . addslashes($this->_rec->val('job_id')) . "' and\n\t\t\t\t\ttml.webpage_id is not null");
$missedWebpageIds = array();
while ($row = mysql_fetch_assoc($res)) {
$missedWebpageIds[$row['webpage_id']][] = $row;
}
@mysql_free_result($res);
// 1. Get all of the webpages
$res = SweteDb::q("select webpage_id from job_inputs_webpages where job_id='" . addslashes($this->_rec->val('job_id')) . "'");
$wpids = array();
while ($row = mysql_fetch_row($res)) {
$wpids[] = $row[0];
}
$site = $this->getSite();
$proxyWriter = $site->getProxyWriter();
$jobWordCount = 0;
@mysql_free_result($res);
foreach ($wpids as $webpageId) {
$webpage = SweteWebpage::loadById($webpageId, $this->_rec->val('source_language'));
if (!$webpage) {
throw new Exception("Could not find webpage with id {$webpageId}");
}
$webpage->setSite($site);
// Use a page sucker to suck all of the resources used by this webpage.
$pageSucker = new SweteJobPageSucker($this);
$pageContent = $webpage->getRecord()->val('webpage_content');
$pageUrl = $site->getSiteUrl() . $webpage->getRecord()->val('webpage_url');
$pageContent = $pageSucker->processHtml($pageContent, $pageUrl);
$translatable = new Dataface_Record('job_translatable', array());
$translatable->setValues(array('job_id' => $this->_rec->val('job_id'), 'content_type' => $webpage->getRecord()->val('last_checked_content_type'), 'full_contents' => $pageContent->save(), 'webpage_id' => $webpageId, 'source_url' => $pageUrl));
//strings from static sites
$strings = array();
$res = SweteDb::q("select `string` from job_inputs_webpages_strings where job_id='" . addslashes($this->_rec->val('job_id')) . "' and webpage_id='" . addslashes($webpageId) . "'");
while ($row = mysql_fetch_row($res)) {
$strings[] = $row[0];
}
@mysql_free_result($res);
// Lets see if there are any other strings that were added individually to this page.
if (isset($missedWebpageIds[$webpageId])) {
foreach ($missedWebpageIds[$webpageId] as $row) {
$strings[] = $row['string'];
}
unset($missedWebpageIds[$webpageId]);
}
// We need to collapse duplicate strings
$uniqueStringIndex = array();
$uniqueStrings = array();
foreach ($strings as $k => $str) {
$nstr = TMTools::normalize($str);
$estr = TMTools::encode($nstr, $temp);
if (!isset($uniqueStringIndex[$estr])) {
$uniqueStrings[] = $str;
$uniqueStringIndex[$estr] = 1;
}
}
$strings = $uniqueStrings;
$translatable->setValue('translatable_contents', '<div>' . implode('</div><div>', $strings) . '</div>');
//set the word count
$pageWordCount = self::getPageWordCount($strings);
$jobWordCount += $pageWordCount;
$translatable->setValue('word_count', $pageWordCount);
// Now we need to get the previous translations
$tmid = $webpage->getTranslationMemoryId(true);
$tm = XFTranslationMemory::loadTranslationMemoryById($tmid);
if (!$tm) {
throw new Exception("Could not find translation memory with id {$tmid}");
}
$dict = $this->extractDictionaryFromHtml($tm, $webpage->getRecord()->val('webpage_content'));
$translatable->setValue('previous_translations', serialize($dict));
$res = $translatable->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage(), $res->getCode());
}
}
// Add the remainder of the missed webpages.
foreach ($missedWebpageIds as $webpageId => $strings) {
$webpage = SweteWebpage::loadById($webpageId, $this->_rec->val('source_language'));
if (!$webpage) {
throw new Exception("Could not find webpage with id {$webpageId}");
}
$webpage->setSite($site);
// Use a page sucker to suck all of the resources used by this webpage.
$pageSucker = new SweteJobPageSucker($this);
$pageContent = $webpage->getRecord()->val('webpage_content');
$pageUrl = $site->getSiteUrl() . $webpage->getRecord()->val('webpage_url');
$pageContent = $pageSucker->processHtml($pageContent, $pageUrl);
$translatable = new Dataface_Record('job_translatable', array());
$translatable->setValues(array('job_id' => $this->_rec->val('job_id'), 'content_type' => $webpage->getRecord()->val('last_checked_content_type'), 'full_contents' => $pageContent->save(), 'webpage_id' => $webpageId, 'source_url' => $pageUrl));
// We need to collapse duplicate strings
$uniqueStringIndex = array();
$uniqueStrings = array();
foreach ($strings as $k => $missedstr) {
$str = $missedstr['string'];
$nstr = TMTools::normalize($str);
$estr = TMTools::normalize(TMTools::encode($nstr, $temp));
if (!isset($uniqueStringIndex[$estr])) {
$uniqueStrings[] = $str;
$uniqueStringIndex[$estr] = 1;
}
}
$strings = $uniqueStrings;
$translatable->setValue('translatable_contents', '<div>' . implode('</div><div>', $strings) . '</div>');
//set the word count
$pageWordCount = self::getPageWordCount($strings);
//strings
$jobWordCount += $pageWordCount;
$translatable->setValue('word_count', $pageWordCount);
// Now we need to get the previous translations
$tmid = $webpage->getTranslationMemoryId(true);
$tm = XFTranslationMemory::loadTranslationMemoryById($tmid);
if (!$tm) {
throw new Exception("Could not find translation memory with id {$tmid}");
}
$dict = $this->extractDictionaryFromHtml($tm, $webpage->getRecord()->val('webpage_content'));
$translatable->setValue('previous_translations', serialize($dict));
$res = $translatable->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage(), $res->getCode());
}
}
// 2. Get all of the http requests without associated webpages.
$res = SweteDb::q("select htl.http_request_log_id, tml.translation_miss_log_id, tml.string, htl.translation_memory_id\n\t\t\t\tfrom \n\t\t\t\t\ttranslation_miss_log tml \n\t\t\t\t\tinner join http_request_log htl on tml.http_request_log_id=htl.http_request_log_id \n\t\t\t\t\tinner join job_inputs_translation_misses jitm on jitm.translation_miss_log_id=tml.translation_miss_log_id\n\t\t\t\t\twhere jitm.job_id='" . addslashes($this->_rec->val('job_id')) . "'");
$hrids = array();
while ($row = mysql_fetch_assoc($res)) {
$hrids[$row['http_request_log_id']][] = $row;
}
//$site = $this->getSite();
//$proxyWriter = $site->getProxyWriter();
@mysql_free_result($res);
foreach ($hrids as $hrid => $tmlids) {
$hrRecord = df_get_record('http_request_log', array('http_request_log_id' => '=' . $hrid));
if (!$hrRecord) {
$ex = new Exception("Cannot add HTTP request to job because it could not be found");
$ex->http_request_log_id = $hrid;
throw $ex;
}
// Use a page sucker to suck all of the resources used by this webpage.
$pageSucker = new SweteJobPageSucker($this);
$pageContent = $hrRecord->val('response_body');
$pageUrl = $hrRecord->val('proxy_request_url');
if (!$pageUrl) {
$ex = new Exception("Failed to add HTTP request to job because it did not have an associated proxy_request_url.");
$ex->http_request_log = $hrid;
throw $ex;
}
$pageUrl = $proxyWriter->unproxifyUrl($pageUrl);
$pageContent = $pageSucker->processHtml($pageContent, $pageUrl)->save();
$translatable = new Dataface_Record('job_translatable', array());
$translatable->setValues(array('job_id' => $this->_rec->val('job_id'), 'content_type' => 'text/html', 'full_contents' => $pageContent, 'webpage_id' => null, 'source_url' => $hrRecord->val('request_url')));
$tmid = null;
$strings = array();
foreach ($tmlids as $tmlid) {
$strings[] = $tmlid['string'];
$tmid = $tmlid['translation_memory_id'];
}
$translatable->setValue('translatable_contents', '<div>' . implode('</div><div>', $strings) . '</div>');
//set the word count
$pageWordCount = self::getPageWordCount($strings);
$jobWordCount += $pageWordCount;
$translatable->setValue('word_count', $pageWordCount);
// Now we need to get the previous translations
//$tmid = $webpage->getTranslationMemoryId(true);
$tm = XFTranslationMemory::loadTranslationMemoryById($tmid);
if (!$tm) {
throw new Exception("Could not find translation memory with id {$tmid}");
}
$dict = $this->extractDictionaryFromHtml($tm, $pageContent);
$translatable->setValue('previous_translations', serialize($dict));
$res = $translatable->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage(), $res->getCode());
}
}
if ($jobWordCount == 0) {
throw new Exception("The job has no translatable content.");
}
$this->getRecord()->setValue('word_count', $jobWordCount);
$this->getRecord()->setValue('compiled', 1);
$res = $this->getRecord()->save();
if (PEAR::isError($res)) {
throw new Exception($res->getMessage());
}
SweteDb::q("commit");
} catch (Exception $ex) {
SweteDb::q("rollback");
throw $ex;
}
}
