/**
* Load the given file via {@link self::processAll()} and {@link self::processRecord()}.
* Optionally truncates (clear) the table before it imports.
*
* @param string $filepath The filepath to use
*
* @return BulkLoader_Result See {@link self::processAll()}
*
* @author Sascha Koehler <*****@*****.**>
* @since 20.07.2011
*/
public function load($filepath)
{
if (!SilvercartPlugin::call($this, 'overwriteLoad', array($filepath), false, 'DataObject')) {
ini_set('max_execution_time', 3600);
increase_memory_limit_to('256M');
//get all instances of the to be imported data object
if ($this->deleteExistingRecords) {
$q = singleton($this->objectClass)->buildSQL();
if (!empty($this->objectClass)) {
$idSelector = $this->objectClass . '."ID"';
} else {
$idSelector = '"ID"';
}
$q->select = array($idSelector);
$ids = $q->execute()->column('ID');
foreach ($ids as $id) {
$obj = DataObject::get_by_id($this->objectClass, $id);
$obj->delete();
$obj->destroy();
unset($obj);
}
}
return $this->processAll($filepath);
}
}
/**
* Collect all changes for the given context.
*/
public function collectChanges($context)
{
increase_time_limit_to();
increase_memory_limit_to();
if (is_callable(array($this->owner, 'objectsToUpdate'))) {
$toUpdate = $this->owner->objectsToUpdate($context);
if ($toUpdate) {
foreach ($toUpdate as $object) {
if (!is_callable(array($this->owner, 'urlsToCache'))) {
continue;
}
$urls = $object->urlsToCache();
if (!empty($urls)) {
$this->toUpdate = array_merge($this->toUpdate, $this->owner->getUrlArrayObject()->addObjects($urls, $object));
}
}
}
}
if (is_callable(array($this->owner, 'objectsToDelete'))) {
$toDelete = $this->owner->objectsToDelete($context);
if ($toDelete) {
foreach ($toDelete as $object) {
if (!is_callable(array($this->owner, 'urlsToCache'))) {
continue;
}
$urls = $object->urlsToCache();
if (!empty($urls)) {
$this->toDelete = array_merge($this->toDelete, $this->owner->getUrlArrayObject()->addObjects($urls, $object));
}
}
}
}
}
function testIncreaseMemoryLimitTo() {
if(!$this->canChangeMemory()) return;
ini_set('memory_limit', '64M');
// It can go up
increase_memory_limit_to('128M');
$this->assertEquals('128M', ini_get('memory_limit'));
// But not down
increase_memory_limit_to('64M');
$this->assertEquals('128M', ini_get('memory_limit'));
// Test the different kinds of syntaxes
increase_memory_limit_to(1024*1024*200);
$this->assertEquals(1024*1024*200, ini_get('memory_limit'));
increase_memory_limit_to('409600K');
$this->assertEquals('409600K', ini_get('memory_limit'));
increase_memory_limit_to('1G');
// If memory limit was left at 409600K, that means that the current testbox doesn't have
// 1G of memory available. That's okay; let's not report a failure for that.
if(ini_get('memory_limit') != '409600K') {
$this->assertEquals('1G', ini_get('memory_limit'));
}
// No argument means unlimited
increase_memory_limit_to();
$this->assertEquals(-1, ini_get('memory_limit'));
}
/**
* Perform migration
*
* @param string $base Absolute base path (parent of assets folder). Will default to BASE_PATH
* @return int Number of files successfully migrated
*/
public function run($base = null)
{
if (empty($base)) {
$base = BASE_PATH;
}
// Check if the File dataobject has a "Filename" field.
// If not, cannot migrate
if (!DB::get_schema()->hasField('File', 'Filename')) {
return 0;
}
// Set max time and memory limit
increase_time_limit_to();
increase_memory_limit_to();
// Loop over all files
$count = 0;
$filenameMap = $this->getFilenameArray();
foreach ($this->getFileQuery() as $file) {
// Get the name of the file to import
$filename = $filenameMap[$file->ID];
$success = $this->migrateFile($base, $file, $filename);
if ($success) {
$count++;
}
}
return $count;
}
public function fire()
{
increase_memory_limit_to();
increase_time_limit_to();
/** @var i18nTextCollector $collector */
$collector = i18nTextCollector::create($this->option('locale'));
if (!$this->option('locale')) {
$this->info('Starting text collection. No Locale specified');
} else {
$this->info('Starting text collection for: ' . $this->option('locale'));
}
$merge = $this->getIsMerge();
if ($merge) {
$this->info('New strings will be merged with existing strings');
}
// Custom writer
$writerName = $this->option('writer');
if ($writerName) {
$writer = Injector::inst()->get($writerName);
$collector->setWriter($writer);
$this->info('Set collector writer to: ' . $writer);
}
// Get restrictions
$restrictModules = $this->option('module') ? explode(',', $this->option('module')) : null;
$this->info('Collecting in modules: ' . $this->option('module'));
// hack untill we have something better
ob_start();
$collector->run($restrictModules, $merge);
$this->warn(ob_get_contents());
ob_get_clean();
$this->info(__CLASS__ . ' completed!');
}
/**
* @param Config_ForClass $config
*/
public function __construct(Config_ForClass $config = null)
{
parent::__construct();
$this->config = $config ?: Config::inst()->forClass(__CLASS__);
if ($memLimit = $this->config->get('memory')) {
increase_memory_limit_to($memLimit);
}
}
/**
* Load the given file via {@link self::processAll()} and {@link self::processRecord()}.
* Optionally truncates (clear) the table before it imports.
*
* @return BulkLoader_Result See {@link self::processAll()}
*/
public function load($filepath)
{
// A small hack to allow model admin import form to work properly
if (!is_array($filepath) && isset($_FILES['_CsvFile'])) {
$filepath = $_FILES['_CsvFile'];
}
if (is_array($filepath)) {
$this->uploadFile = $filepath;
$filepath = $filepath['tmp_name'];
}
increase_time_limit_to();
increase_memory_limit_to('512M');
//get all instances of the to be imported data object
if ($this->deleteExistingRecords) {
DataObject::get($this->objectClass)->removeAll();
}
return $this->processAll($filepath);
}
function testIncreaseMemoryLimitTo()
{
ini_set('memory_limit', '64M');
// It can go up
increase_memory_limit_to('128M');
$this->assertEquals('128M', ini_get('memory_limit'));
// But not down
increase_memory_limit_to('64M');
$this->assertEquals('128M', ini_get('memory_limit'));
// Test the different kinds of syntaxes
increase_memory_limit_to(1024 * 1024 * 200);
$this->assertEquals(1024 * 1024 * 200, ini_get('memory_limit'));
increase_memory_limit_to('409600K');
$this->assertEquals('409600K', ini_get('memory_limit'));
increase_memory_limit_to('1G');
$this->assertEquals('1G', ini_get('memory_limit'));
// No argument means unlimited
increase_memory_limit_to();
$this->assertEquals(-1, ini_get('memory_limit'));
}
public function run($request)
{
set_time_limit(0);
increase_memory_limit_to();
Subsite::$disable_subsite_filter = true;
$mainConfig = SiteConfig::current_site_config();
$mainConfig->compileStyles();
DB::alteration_message("Compile styles for main site");
$subsites = Subsite::get();
foreach ($subsites as $subsite) {
$subsiteConfig = SiteConfig::get()->filter('SubsiteID', $subsite->ID)->first();
if (!$subsiteConfig) {
DB::alteration_message("No config for subsite " . $subsite->ID, "error");
continue;
}
$subsiteConfig->compileStyles();
DB::alteration_message("Compile styles for subsite " . $subsite->ID);
}
DB::alteration_message("All done");
}
/**
* Uses {@link Director::test()} to perform in-memory HTTP requests
* on the passed-in URLs.
*
* @param array $urls Relative URLs
* @return array Result, keyed by URL. Keys:
* - "statuscode": The HTTP status code
* - "redirect": A redirect location (if applicable)
* - "path": The filesystem path where the cache has been written
*/
public function publishPages($urls)
{
$result = array();
//nest the config so we can make changes to the config and revert easily
Config::nest();
// Do we need to map these?
// Detect a numerically indexed arrays
if (is_numeric(join('', array_keys($urls)))) {
$urls = $this->urlsToPaths($urls);
}
// This can be quite memory hungry and time-consuming
// @todo - Make a more memory efficient publisher
increase_time_limit_to();
increase_memory_limit_to();
// Set the appropriate theme for this publication batch.
// This may have been set explicitly via StaticPublisher::static_publisher_theme,
// or we can use the last non-null theme.
$customTheme = Config::inst()->get('StaticPublisher', 'static_publisher_theme');
if ($customTheme) {
Config::inst()->update('SSViewer', 'theme', $customTheme);
}
// Ensure that the theme that is set gets used.
Config::inst()->update('SSViewer', 'theme_enabled', true);
$staticBaseUrl = Config::inst()->get('FilesystemPublisher', 'static_base_url');
if ($staticBaseUrl) {
Config::inst()->update('Director', 'alternate_base_url', $staticBaseUrl);
}
if ($this->fileExtension == 'php') {
Config::inst()->update('SSViewer', 'rewrite_hash_links', 'php');
}
if (Config::inst()->get('StaticPublisher', 'echo_progress')) {
echo $this->class . ": Publishing to " . $staticBaseUrl . "\n";
}
$files = array();
$i = 0;
$totalURLs = sizeof($urls);
foreach ($urls as $url => $path) {
$origUrl = $url;
$result[$origUrl] = array('statuscode' => null, 'redirect' => null, 'path' => null);
$i++;
if ($url && !is_string($url)) {
user_error("Bad url:" . var_export($url, true), E_USER_WARNING);
continue;
}
if (Config::inst()->get('StaticPublisher', 'echo_progress')) {
echo " * Publishing page {$i}/{$totalURLs}: {$url}\n";
flush();
}
Requirements::clear();
if ($url == "") {
$url = "/";
}
if (Director::is_relative_url($url)) {
$url = Director::absoluteURL($url);
}
$response = Director::test(str_replace('+', ' ', $url));
if (!$response) {
continue;
}
if ($response) {
$result[$origUrl]['statuscode'] = $response->getStatusCode();
}
Requirements::clear();
singleton('DataObject')->flushCache();
// Check for ErrorPages generating output - we want to handle this in a special way below.
$isErrorPage = false;
$pageObject = null;
if ($response && is_object($response) && (int) $response->getStatusCode() >= 400) {
$pageObject = SiteTree::get_by_link($url);
if ($pageObject && $pageObject instanceof ErrorPage) {
$isErrorPage = true;
}
}
// Skip any responses with a 404 status code unless it's the ErrorPage itself.
if (!$isErrorPage && is_object($response) && $response->getStatusCode() == '404') {
continue;
}
// Generate file content
// PHP file caching will generate a simple script from a template
if ($this->fileExtension == 'php') {
if (is_object($response)) {
if ($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
$content = $this->generatePHPCacheRedirection($response->getHeader('Location'));
} else {
$content = $this->generatePHPCacheFile($response->getBody(), HTTP::get_cache_age(), date('Y-m-d H:i:s'), $response->getHeader('Content-Type'));
}
} else {
$content = $this->generatePHPCacheFile($response . '', HTTP::get_cache_age(), date('Y-m-d H:i:s'), $response->getHeader('Content-Type'));
}
// HTML file caching generally just creates a simple file
} else {
if (is_object($response)) {
if ($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
$absoluteURL = Director::absoluteURL($response->getHeader('Location'));
$result[$origUrl]['redirect'] = $response->getHeader('Location');
$content = "<meta http-equiv=\"refresh\" content=\"2; URL={$absoluteURL}\">";
} else {
$content = $response->getBody();
}
} else {
$content = $response . '';
}
}
if (Config::inst()->get('StaticPublisher', 'include_caching_metadata')) {
$content = str_replace('</html>', sprintf("</html>\n\n<!-- %s -->", implode(" ", $this->getMetadata($url))), $content);
}
if (!$isErrorPage) {
$files[$origUrl] = array('Content' => $content, 'Folder' => dirname($path) . '/', 'Filename' => basename($path));
} else {
// Generate a static version of the error page with a standardised name, so they can be plugged
// into catch-all webserver statements such as Apache's ErrorDocument.
$code = (int) $response->getStatusCode();
$files[$origUrl] = array('Content' => $content, 'Folder' => dirname($path) . '/', 'Filename' => "error-{$code}.html");
}
}
//return config to its previous state
Config::unnest();
$base = BASE_PATH . "/{$this->destFolder}";
foreach ($files as $origUrl => $file) {
Filesystem::makeFolder("{$base}/{$file['Folder']}");
$path = "{$base}/{$file['Folder']}{$file['Filename']}";
$result[$origUrl]['path'] = $path;
if (isset($file['Content'])) {
$fh = fopen($path, "w");
fwrite($fh, $file['Content']);
fclose($fh);
} else {
if (isset($file['Copy'])) {
copy($file['Copy'], $path);
}
}
}
return $result;
}
public function load($filepath)
{
ini_set('max_execution_time', 3600);
increase_memory_limit_to('512M');
//get all instances of the to be imported data object
if ($this->deleteExistingRecords) {
$q = singleton($this->objectClass)->buildSQL();
$q->select = array('"ID"');
$ids = $q->execute()->column('ID');
foreach ($ids as $id) {
$obj = DataObject::get_by_id($this->objectClass, $id);
$obj->delete();
$obj->destroy();
unset($obj);
}
}
return $this->processAll($filepath);
}
/**
* Uses {@link Director::test()} to perform in-memory HTTP requests
* on the passed-in URLs.
*
* @param array $urls Relative URLs
* @return array Result, keyed by URL. Keys:
* - "statuscode": The HTTP status code
* - "redirect": A redirect location (if applicable)
* - "path": The filesystem path where the cache has been written
*/
public function publishPages($urls)
{
$result = array();
// Do we need to map these?
// Detect a numerically indexed arrays
if (is_numeric(join('', array_keys($urls)))) {
$urls = $this->urlsToPaths($urls);
}
// This can be quite memory hungry and time-consuming
// @todo - Make a more memory efficient publisher
increase_time_limit_to();
increase_memory_limit_to();
Config::inst()->nest();
// Set the appropriate theme for this publication batch.
// This may have been set explicitly via StaticPublisher::static_publisher_theme,
// or we can use the last non-null theme.
$customTheme = Config::inst()->get('FilesystemPublisher', 'static_publisher_theme');
if ($customTheme) {
Config::inst()->update('SSViewer', 'theme', $customTheme);
}
// Ensure that the theme that is set gets used.
Config::inst()->update('SSViewer', 'theme_enabled', true);
$currentBaseURL = Director::baseURL();
$staticBaseUrl = Config::inst()->get('FilesystemPublisher', 'static_base_url');
if ($this->fileExtension == 'php') {
Config::inst()->update('SSViewer', 'rewrite_hash_links', 'php');
}
if (Config::inst()->get('FilesystemPublisher', 'echo_progress')) {
echo $this->class . ": Publishing to " . $staticBaseUrl . "\n";
}
$files = array();
$i = 0;
$totalURLs = sizeof($urls);
foreach ($urls as $url => $path) {
$origUrl = $url;
$result[$origUrl] = array('statuscode' => null, 'redirect' => null, 'path' => null);
$i++;
if ($url && !is_string($url)) {
user_error("Bad url:" . var_export($url, true), E_USER_WARNING);
continue;
}
if (Config::inst()->get('FilesystemPublisher', 'echo_progress')) {
echo " * Publishing page {$i}/{$totalURLs}: {$url}\n";
flush();
}
Requirements::clear();
if ($url == "") {
$url = "/";
}
if (Director::is_relative_url($url)) {
$url = Director::absoluteURL($url);
}
$sanitizedURL = URLArrayObject::sanitize_url($url);
$response = Director::test(str_replace('+', ' ', $sanitizedURL));
// Prevent empty static cache files from being written
if (is_object($response) && !$response->getBody()) {
SS_Log::log(new Exception('Prevented blank static cache page write for: ' . $path), SS_Log::NOTICE);
continue;
}
if (!$response) {
continue;
}
if ($response) {
$result[$origUrl]['statuscode'] = $response->getStatusCode();
}
Requirements::clear();
singleton('DataObject')->flushCache();
// Check for ErrorPages generating output - we want to handle this in a special way below.
$isErrorPage = false;
$pageObject = null;
if ($response && is_object($response) && (int) $response->getStatusCode() >= 400) {
$obj = $this->owner->getUrlArrayObject()->getObject($url);
if ($obj && $obj instanceof ErrorPage) {
$isErrorPage = true;
}
}
// Skip any responses with a 404 status code unless it's the ErrorPage itself.
if (!$isErrorPage && is_object($response) && $response->getStatusCode() == '404') {
continue;
}
// Generate file content.
// PHP file caching will generate a simple script from a template
if ($this->fileExtension == 'php') {
if (is_object($response)) {
if ($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
$content = $this->generatePHPCacheRedirection($response->getHeader('Location'));
} else {
$content = $this->generatePHPCacheFile($response->getBody(), HTTP::get_cache_age(), date('Y-m-d H:i:s'), $response->getHeader('Content-Type'));
}
} else {
$content = $this->generatePHPCacheFile($response . '', HTTP::get_cache_age(), date('Y-m-d H:i:s'), $response->getHeader('Content-Type'));
}
// HTML file caching generally just creates a simple file
} else {
if (is_object($response)) {
if ($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
$absoluteURL = Director::absoluteURL($response->getHeader('Location'));
$result[$origUrl]['redirect'] = $response->getHeader('Location');
$content = "<meta http-equiv=\"refresh\" content=\"2; URL={$absoluteURL}\">";
} else {
$content = $response->getBody();
}
} else {
$content = $response . '';
}
}
if (Config::inst()->get('FilesystemPublisher', 'include_caching_metadata')) {
$content = str_replace('</html>', sprintf("</html>\n\n<!-- %s -->", implode(" ", $this->getMetadata($url))), $content);
}
if (!$isErrorPage) {
$files[$origUrl] = array('Content' => $content, 'Folder' => dirname($path) . '/', 'Filename' => basename($path));
} else {
// Generate a static version of the error page with a standardised name, so they can be plugged
// into catch-all webserver statements such as Apache's ErrorDocument.
$code = (int) $response->getStatusCode();
$files[$origUrl] = array('Content' => $content, 'Folder' => dirname($path) . '/', 'Filename' => "error-{$code}.html");
}
// Add externals
/*
$externals = $this->externalReferencesFor($content);
if($externals) foreach($externals as $external) {
// Skip absolute URLs
if(preg_match('/^[a-zA-Z]+:\/\//', $external)) continue;
// Drop querystring parameters
$external = strtok($external, '?');
if(file_exists("../" . $external)) {
// Break into folder and filename
if(preg_match('/^(.*\/)([^\/]+)$/', $external, $matches)) {
$files[$external] = array(
"Copy" => "../$external",
"Folder" => $matches[1],
"Filename" => $matches[2],
);
} else {
user_error("Can't parse external: $external", E_USER_WARNING);
}
} else {
$missingFiles[$external] = true;
}
}*/
}
if ($this->fileExtension == 'php') {
Config::inst()->update('SSViewer', 'rewrite_hash_links', true);
}
$base = BASE_PATH . "/{$this->destFolder}";
foreach ($files as $origUrl => $file) {
Filesystem::makeFolder("{$base}/{$file['Folder']}");
$path = "{$base}/{$file['Folder']}{$file['Filename']}";
$result[$origUrl]['path'] = $path;
if (isset($file['Content'])) {
$fh = fopen($path, "w");
fwrite($fh, $file['Content']);
fclose($fh);
} else {
if (isset($file['Copy'])) {
copy($file['Copy'], $path);
}
}
}
Config::inst()->unnest();
return $result;
}
public function process()
{
self::$config = $this->config();
if (!self::$config->wkHtmlToPdfPath) {
throw new Exception("You must provide a path for WkHtmlToPdf in your sites configuration.");
}
if (!self::$config->emailAddress) {
throw new Exception("You must provide an email address to send from in your sites configuration.");
}
increase_memory_limit_to('1024M');
set_time_limit(0);
$sites = LinkCheckSite::get();
$outputDir = BASE_PATH . DIRECTORY_SEPARATOR . "silverstripe-linkcheck/runs/";
$filesCreated = array();
// build the crawler
chdir(__DIR__ . "/../thirdparty");
exec("javac " . self::$crawler . " " . self::$linkStats . " && " . "javac " . self::$linkProject);
if ($sites) {
foreach ($sites as $site) {
echo "Checking " . $site->SiteURL . "\r\n";
$url = $site->SiteURL;
// if the output directory doesn't exist for the run, create it
if (!file_exists($outputDir . str_replace("http://", "", $url))) {
mkdir($outputDir . str_replace("http://", "", $url));
}
$filename = date("Y-m-d") . '-' . rand(0, 1000) . ".html";
$filepath = $outputDir . str_replace("http://", "", $url) . '/';
// execute the crawler
exec("java Project {$url} " . $filepath . $filename . " 10 1000");
$filesCreated[$site->ID]['FilePath'] = $filepath;
$filesCreated[$site->ID]['FileName'] = $filename;
$filesCreated[$site->ID]['SiteName'] = $site->SiteName;
$filesCreated[$site->ID]['ID'] = $site->ID;
$filesCreated[$site->ID]['URL'] = $url;
$emailRecipients = $site->EmailRecipients();
if ($emailRecipients) {
foreach ($emailRecipients as $recipient) {
$filesCreated[$site->ID]['Email'][] = $recipient->Email;
}
}
}
foreach ($filesCreated as $file) {
Folder::find_or_make("LinkCheck" . DIRECTORY_SEPARATOR . $file['SiteName'] . DIRECTORY_SEPARATOR);
$pdfPath = "assets" . DIRECTORY_SEPARATOR . "LinkCheck" . DIRECTORY_SEPARATOR . $file['SiteName'] . DIRECTORY_SEPARATOR;
$pdfFullPath = BASE_PATH . DIRECTORY_SEPARATOR . $pdfPath;
$pdfName = str_replace("html", "pdf", $file['FileName']);
$generator = new WkHtml\Generator(new \Knp\Snappy\Pdf(self::$config->wkHtmlToPdfPath), new WkHtml\Input\String(file_get_contents($file['FilePath'] . $file['FileName'])), new WkHtml\Output\File($pdfFullPath . $pdfName, 'application/pdf'));
$generator->process();
$site = LinkCheckSite::get()->byID($file['ID']);
$pdfUpload = new File();
$pdfUpload->Title = $file['SiteName'] . '-' . $pdfName;
$pdfUpload->Filename = $pdfPath . $pdfName;
$pdfUpload->write();
$linkCheckRun = new LinkCheckRun();
$linkCheckRun->LinkCheckFileID = $pdfUpload->ID;
$linkCheckRun->LinkCheckSiteID = $site->ID;
$linkCheckRun->write();
$site->LinkCheckRuns()->add($linkCheckRun);
foreach ($file['Email'] as $emailAddress) {
$email = new Email();
$email->to = $emailAddress;
$email->from = $this->config()->emailAddress;
$email->subject = $file['SiteName'] . " link check run";
$email->body = "Site Link Check Run for {$file['URL']} on " . date("Y/m/d");
$email->attachFile($pdfPath . $pdfName, "linkcheck.pdf");
$email->send();
}
unlink($file['FilePath'] . $file['FileName']);
}
}
}
/**
* Define the temporary folder if it wasn't defined yet
*/
if (!defined('TEMP_FOLDER')) {
define('TEMP_FOLDER', getTempFolder());
}
/**
* Priorities definition. These constants are used in calls to _t() as an optional argument
*/
define('PR_HIGH', 100);
define('PR_MEDIUM', 50);
define('PR_LOW', 10);
/**
* Ensure we have enough memory
*/
increase_memory_limit_to('64M');
///////////////////////////////////////////////////////////////////////////////
// INCLUDES
require_once "core/ManifestBuilder.php";
require_once "core/ClassInfo.php";
require_once 'core/Object.php';
require_once 'core/control/Director.php';
require_once 'filesystem/Filesystem.php';
require_once "core/Session.php";
///////////////////////////////////////////////////////////////////////////////
// MANIFEST
/**
* Include the manifest
*/
ManifestBuilder::include_manifest();
/**
/**
* Either creates or updates a record in the index.
*
* @param Searchable $record
* @return \Elastica\Response
*/
public function index($record)
{
if (!$this->indexingMemorySet && $this->indexingMemory) {
if ($this->indexingMemory == 'unlimited') {
increase_memory_limit_to();
} else {
increase_memory_limit_to($this->indexingMemory);
}
$this->indexingMemorySet = true;
}
try {
$document = $record->getElasticaDocument();
$type = $record->getElasticaType();
$index = $this->getIndex();
$response = $index->getType($type)->addDocument($document);
$index->refresh();
return $response;
} catch (\Exception $e) {
if ($this->logger) {
$this->logger->warning($e->getMessage());
}
}
}
function run($request)
{
set_time_limit(1200);
increase_memory_limit_to(-1);
$nextGetStatement = "";
//can we do the next step?
//IN general yes, but if the current one is not finished yet, then we do it again.
$canDoNext = true;
if (isset($_REQUEST["limit"])) {
$this->limit = intval($_REQUEST["limit"]);
}
if (isset($_REQUEST["start"])) {
$this->start = intval($_REQUEST["start"]);
}
//what is the current step?
$currentMethod = isset($_REQUEST["action"]) ? $_REQUEST["action"] : "";
if (in_array($currentMethod, $this->listOfMigrationTasks)) {
//are we doing the same one for a different limti?
if ($this->start) {
$this->DBAlterationMessageNow("this task is broken down into baby steps - we are now starting at .... " . $this->start . " processing " . $this->limit, "created");
}
$nextLimit = $this->{$currentMethod}();
if ($canDoNext && $nextLimit) {
$canDoNext = false;
//NEXT OPTION 1: do again with new limit
$nextGetStatement = "?action=" . $currentMethod . "&start=" . $nextLimit;
$nextDescription = "run next batch ...";
}
}
if ($canDoNext && !$nextGetStatement) {
//NEXT OPTION 2: start from the beginning
$nextGetStatement = "?fullmigration=1&action=" . $this->listOfMigrationTasks[0];
$nextDescription = "Start Migration by clicking on <i>'Next'</i> (this link) until all tasks have been completed.";
}
//retrieve data...
$this->retrieveInfoOnly = true;
$html = "";
$createListOfActions = false;
if (!$currentMethod) {
$createListOfActions = true;
}
if ($createListOfActions) {
$html .= "\r\n\t\t\t<p>Always make a backup of your database before running any migration tasks.</p>\r\n\t\t\t<ul>";
}
foreach ($this->listOfMigrationTasks as $key => $task) {
$explanation = $this->{$task}();
$explanation = str_replace(array("<h1>", "</h1>", "<p>", "</p>"), array("<strong>", "</strong>: ", "<span style=\"color: grey;\">", "</span>"), $explanation);
if ($task == $currentMethod) {
if ($canDoNext) {
$keyPlusOne = $key + 1;
if (isset($this->listOfMigrationTasks[$keyPlusOne]) && isset($_REQUEST["fullmigration"])) {
//NEXT OPTION 3: next action!
$action = $this->listOfMigrationTasks[$keyPlusOne];
$nextGetStatement = "?action=" . $action;
$nextDescription = $this->{$action}();
$nextDescription = str_replace(array("<h1>", "</h1>", "<p>", "</p>"), array("<strong>", "</strong>: ", "<span style=\"color: grey;\">", "</span>"), $nextDescription);
} else {
//NEXT OPTION 4: we have done all of them - no more next option...
$nextGetStatement = "";
$nextDescription = "";
}
}
}
if ($createListOfActions) {
$html .= "<li><a href=\"/dev/ecommerce/ecommercetaskmigration/?action=" . $task . "\">{$explanation} </a></li>";
}
}
if ($createListOfActions) {
$html .= "</ul>";
}
if ($nextGetStatement) {
$nextLink = "/dev/ecommerce/ecommercetaskmigration/" . $nextGetStatement;
if (isset($_REQUEST["fullmigration"])) {
$nextLink .= "&fullmigration=1";
}
echo "\r\n\t\t\t\t<hr style=\"margin-top: 50px;\"/>\r\n\t\t\t\t<h3><a href=\"{$nextLink}\">NEXT: {$nextDescription}</a></h3>";
if ($currentMethod) {
echo "\r\n\t\t\t\t<div style=\"width: 400px; height: 20px; padding-top: 20px; font-size: 11px; background: url(/ecommerce/images/loading.gif) no-repeat top left transparent\">\r\n\t\t\t\t\tNext step, if any - will load automatically in ten seconds.\r\n\t\t\t\t</div>\r\n\t\t\t\t<script type=\"text/javascript\">\r\n\t\t\t\t\tvar t = window.setTimeout(\r\n\t\t\t\t\t\tfunction(){\r\n\t\t\t\t\t\t\twindow.location = '{$nextLink}';\r\n\t\t\t\t\t\t},\r\n\t\t\t\t\t\t500\r\n\t\t\t\t\t);\r\n\t\t\t\t</script>\r\n\t\t\t\t<hr style=\"margin-bottom: 500px;\"/>\r\n\t\t\t";
}
}
echo $html;
}
public function exportTo($directory)
{
$directory = rtrim($directory, '/');
$links = $this->urlsToPaths($this->getLinks());
$files = array();
increase_time_limit_to();
increase_memory_limit_to();
// Make the output directory if it doesn't exist.
if (!is_dir($directory)) {
mkdir($directory, Filesystem::$folder_create_mask, true);
}
if ($this->theme) {
SSViewer::set_theme($this->theme);
} else {
SSViewer::set_theme(SSViewer::current_custom_theme());
}
if ($this->baseUrl && !$this->makeRelative) {
$originalBaseUrl = Director::baseURL();
Director::setBaseURL($this->baseUrl);
}
// Loop through each link that we're publishing, and create a static
// html file for each.
foreach ($links as $link => $path) {
Requirements::clear();
singleton('DataObject')->flushCache();
$response = Director::test($link);
$target = $directory . '/' . $path;
if (is_object($response)) {
if ($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
$absoluteURL = Director::absoluteURL($response->getHeader('Location'));
$content = "<meta http-equiv=\"refresh\" content=\"2; URL={$absoluteURL}\">";
} else {
$content = $response->getBody();
}
} else {
$content = (string) $response;
}
// Find any external content references inside the response, and add
// them to the copy array.
$externals = $this->externalReferencesFor($content);
if ($externals) {
foreach ($externals as $external) {
if (!Director::is_site_url($external)) {
continue;
}
$external = strtok($external, '?');
$external = Director::makeRelative($external);
if (file_exists(BASE_PATH . '/' . $external)) {
$files["{$directory}/{$external}"] = array('copy', BASE_PATH . '/' . $external);
}
}
}
// Append any anchor links which point to a relative site link
// with a .html extension.
$base = preg_quote(Director::baseURL());
$content = preg_replace('~<a(.+?)href="(' . $base . '[^"]*?)/?"~i', '<a$1href="$2.html"', $content);
$content = str_replace('/.html', '/index.html', $content);
// If we want to rewrite links to relative, then determine how many
// levels deep we are and rewrite the relevant attributes globally.
// Also, string the base tag.
if ($this->makeRelative) {
$content = preg_replace('~(src|href)="' . Director::protocolAndHost() . '~i', '$1="', $content);
if (($trimmed = trim($link, '/')) && strpos($trimmed, '/')) {
$prepend = str_repeat('../', substr_count($trimmed, '/'));
} else {
$prepend = './';
}
$base = preg_quote(Director::baseURL());
$content = preg_replace('~(href|src)="' . $base . '~i', '$1="' . $prepend, $content);
$content = preg_replace('~<base href="([^"]+)" />~', '', $content);
$content = preg_replace('~<base href="([^"]+)"><!--[if lte IE 6]></base><![endif]-->~', '', $content);
}
$files[$target] = array('create', $content);
}
// If we currently have a theme active, then copy all the theme
// assets across to the site.
if ($theme = SSViewer::current_theme()) {
$stack = array(THEMES_PATH . '/' . $theme);
// Build up a list of every file present in the current theme
// which is not a .ss template, and add it to the files array
while ($path = array_pop($stack)) {
foreach (scandir($path) as $file) {
if ($file[0] == '.' || $file[0] == '_') {
continue;
}
if (is_dir("{$path}/{$file}")) {
$stack[] = "{$path}/{$file}";
} else {
if (substr($file, -3) != '.ss') {
$loc = "{$path}/{$file}";
$to = $directory . '/' . substr($loc, strlen(BASE_PATH) + 1);
$files[$to] = array('copy', $loc);
}
}
}
}
}
// If theres a favicon.ico file in the site root, copy it across
if (file_exists(BASE_PATH . '/favicon.ico')) {
$files["{$directory}/favicon.ico"] = array('copy', BASE_PATH . '/favicon.ico');
}
// Copy across or create all the files that have been generated.
foreach ($files as $to => $from) {
list($mode, $content) = $from;
if (!is_dir(dirname($to))) {
mkdir(dirname($to), Filesystem::$folder_create_mask, true);
}
if ($mode == 'create') {
file_put_contents($to, $content);
} else {
if (!file_exists($to)) {
copy($content, $to);
}
}
}
if ($this->baseUrl && !$this->makeRelative) {
Director::setBaseURL($originalBaseUrl);
}
}
public function run($request)
{
increase_time_limit_to();
increase_memory_limit_to();
$formId = $request->getVar('form');
$force = $request->getVar('force') || false;
if ($request->getVar('delete') && Director::isDev()) {
$submissions = TypeformSubmission::get();
if ($formId) {
$submissions = $submissions->filter('ParentID', $formId);
}
foreach ($submissions as $submission) {
$submission->delete();
}
}
foreach ($this->config()->typeform_classes as $class) {
$forms = DataObject::get($class);
if (Director::is_cli()) {
echo "Syncing " . $class . " forms\n";
} else {
echo "<p>Syncing " . $class . " forms</p>";
}
if (!$formId) {
if (Director::is_cli()) {
echo $forms->count() . " found\n";
} else {
echo "<p>" . $forms->count() . " found</p>";
}
}
foreach ($forms as $form) {
$key = null;
if ($form->hasMethod('getTypeformUid')) {
$key = $form->getTypeformUid();
}
if ($key && $formId && $form->ID !== $formId) {
if (Director::is_cli()) {
echo sprintf("* Skipping %s\n", $form->Title);
} else {
echo sprintf("<li>Skipping %s</li>", $form->Title);
}
continue;
}
if ($key) {
$fetch = new SyncTypeformSubmissions_Single($key);
$results = $fetch->syncComments($form, $force);
$total = $results['total'];
$synced = $results['synced'];
if (Director::is_cli()) {
echo sprintf("* %d new synced submissions out of %d total for %s\n", $synced, $total, $form->Title);
} else {
echo sprintf("<li>%d new synced submissions out of %d for %s</li>", $synced, $total, $form->Title);
}
} else {
if (Director::is_cli()) {
echo sprintf("* No valid key for %s\n", $form->Title);
} else {
echo sprintf("<li>No valid key for %s</li>", $form->Title);
}
}
}
}
}
/**
*
*/
public function run($request)
{
increase_time_limit_to(3600);
increase_memory_limit_to('512M');
if ($request->param("forreal") || isset($_GET["forreal"]) && $_GET["forreal"] == 1) {
$this->forreal = true;
}
if ($this->forreal) {
$this->reset();
}
$this->readFile();
$this->createProducts();
$this->findVariations();
if ($this->forreal) {
$this->createVariations();
$this->getExtraDataForVariations();
} else {
$this->showData();
}
}
public function load($filepath)
{
increase_time_limit_to(3600);
increase_memory_limit_to('512M');
//get all instances of the to be imported data object
if ($this->deleteExistingRecords) {
DataObject::get($this->objectClass)->removeAll();
}
return $this->processAll($filepath);
}
function publishPages($urls)
{
// Do we need to map these?
// Detect a numerically indexed arrays
if (is_numeric(join('', array_keys($urls)))) {
$urls = $this->urlsToPaths($urls);
}
// This can be quite memory hungry and time-consuming
// @todo - Make a more memory efficient publisher
increase_time_limit_to();
increase_memory_limit_to();
// Set the appropriate theme for this publication batch.
// This may have been set explicitly via StaticPublisher::static_publisher_theme,
// or we can use the last non-null theme.
if (!StaticPublisher::static_publisher_theme()) {
SSViewer::set_theme(SSViewer::current_custom_theme());
} else {
SSViewer::set_theme(StaticPublisher::static_publisher_theme());
}
$currentBaseURL = Director::baseURL();
if (self::$static_base_url) {
Director::setBaseURL(self::$static_base_url);
}
if ($this->fileExtension == 'php') {
SSViewer::setOption('rewriteHashlinks', 'php');
}
if (StaticPublisher::echo_progress()) {
echo $this->class . ": Publishing to " . self::$static_base_url . "\n";
}
$files = array();
$i = 0;
$totalURLs = sizeof($urls);
foreach ($urls as $url => $path) {
if (self::$static_base_url) {
Director::setBaseURL(self::$static_base_url);
}
$i++;
if ($url && !is_string($url)) {
user_error("Bad url:" . var_export($url, true), E_USER_WARNING);
continue;
}
if (StaticPublisher::echo_progress()) {
echo " * Publishing page {$i}/{$totalURLs}: {$url}\n";
flush();
}
Requirements::clear();
if ($url == "") {
$url = "/";
}
if (Director::is_relative_url($url)) {
$url = Director::absoluteURL($url);
}
$response = Director::test(str_replace('+', ' ', $url));
Requirements::clear();
singleton('DataObject')->flushCache();
//skip any responses with a 404 status code. We don't want to turn those into statically cached pages
if (!$response || $response->getStatusCode() == '404') {
continue;
}
// Generate file content
// PHP file caching will generate a simple script from a template
if ($this->fileExtension == 'php') {
if (is_object($response)) {
if ($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
$content = $this->generatePHPCacheRedirection($response->getHeader('Location'));
} else {
$content = $this->generatePHPCacheFile($response->getBody(), HTTP::get_cache_age(), date('Y-m-d H:i:s'));
}
} else {
$content = $this->generatePHPCacheFile($response . '', HTTP::get_cache_age(), date('Y-m-d H:i:s'));
}
// HTML file caching generally just creates a simple file
} else {
if (is_object($response)) {
if ($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
$absoluteURL = Director::absoluteURL($response->getHeader('Location'));
$content = "<meta http-equiv=\"refresh\" content=\"2; URL={$absoluteURL}\">";
} else {
$content = $response->getBody();
}
} else {
$content = $response . '';
}
}
$files[] = array('Content' => $content, 'Folder' => dirname($path) . '/', 'Filename' => basename($path));
// Add externals
/*
$externals = $this->externalReferencesFor($content);
if($externals) foreach($externals as $external) {
// Skip absolute URLs
if(preg_match('/^[a-zA-Z]+:\/\//', $external)) continue;
// Drop querystring parameters
$external = strtok($external, '?');
if(file_exists("../" . $external)) {
// Break into folder and filename
if(preg_match('/^(.*\/)([^\/]+)$/', $external, $matches)) {
$files[$external] = array(
"Copy" => "../$external",
"Folder" => $matches[1],
"Filename" => $matches[2],
);
} else {
user_error("Can't parse external: $external", E_USER_WARNING);
}
} else {
$missingFiles[$external] = true;
}
}*/
}
if (self::$static_base_url) {
Director::setBaseURL($currentBaseURL);
}
if ($this->fileExtension == 'php') {
SSViewer::setOption('rewriteHashlinks', true);
}
$base = BASE_PATH . "/{$this->destFolder}";
foreach ($files as $file) {
Filesystem::makeFolder("{$base}/{$file['Folder']}");
if (isset($file['Content'])) {
$fh = fopen("{$base}/{$file['Folder']}{$file['Filename']}", "w");
fwrite($fh, $file['Content']);
fclose($fh);
} else {
if (isset($file['Copy'])) {
copy($file['Copy'], "{$base}/{$file['Folder']}{$file['Filename']}");
}
}
}
}
/**
* Unpublish a list of URLs
*
* @param array $urls
* The URLs to unpublish
* @param string $keyPrefix
* The 'prefix' of these URLs as stored in cache. In multisite systems this is generally the
* subsite's primary domain, but may be something more complex if publishing the same content for
* multiple domains
*/
function unpublishUrls($urls)
{
global $PROXY_CACHE_HOSTMAP;
// Do we need to map these?
// Detect a numerically indexed arrays
if (is_numeric(join('', array_keys($urls)))) {
$urls = $this->urlsToPaths($urls);
}
// This can be quite memory hungry and time-consuming
// @todo - Make a more memory efficient publisher
increase_time_limit_to();
increase_memory_limit_to();
$cache = $this->getCache();
foreach ($urls as $url => $path) {
$baseUrlSrc = $this->staticBaseUrl ? $this->staticBaseUrl : $url;
$urlBits = parse_url($baseUrlSrc);
if (!isset($urlBits['host'])) {
$urlBits = parse_url(Director::absoluteBaseURL());
}
$domain = isset($urlBits['host']) ? $urlBits['host'] : (isset($_SERVER['HOST_NAME']) ? $_SERVER['HOST_NAME'] : '');
$key = $domain . '/' . ltrim($path, '/');
$cache->expire($key);
if ($domain && isset($PROXY_CACHE_HOSTMAP) && isset($PROXY_CACHE_HOSTMAP[$domain])) {
$hosts = $PROXY_CACHE_HOSTMAP[$domain];
foreach ($hosts as $otherDomain) {
$key = $otherDomain . '/' . ltrim($path, '/');
$cache->expire($key);
}
}
}
}
/**
* -----------------------------------------------------------------------------------
* Overload the load method of the BulkLoader class.
* Optionally empty all related tables if required.
*
* @return BulkLoader_Result See {@link self::processAll()}
* -----------------------------------------------------------------------------------
*/
public function load($filepath)
{
// Set the execution time and memory limit
ini_set('max_execution_time', 3600);
increase_memory_limit_to('512M');
// Automatically remove all records before uploading new ones
$this->deleteExistingRecords = true;
// If the existing tables need to be emptied
if ($this->deleteExistingRecords) {
// ----------------------------------------------------------
// START : Get MenuSection table data for later use
// ----------------------------------------------------------
// Get the MenuPageID
$result = DataObject::get_one('MenuSection');
// If there is a record
if ($result) {
// Set it into a session var
Session::set('MenuPageID', $result->MenuPageID);
// Otherwise
} else {
// Set the session var to 0
Session::set('MenuPageID', 0);
}
// Get any SectionImageIDs
$result = DataObject::get('MenuSection');
// If there are records returned
if ($result) {
// Add them into an array with the Title as the key
$aImageIDs = array();
foreach ($result as $record) {
$aImageIDs[strtolower($record->Title)] = $record->SectionImageID;
}
// Set the array into a session var
Session::set('SectionImageIDs', $aImageIDs);
}
// ----------------------------------------------------------
// END : Get MenuSection table data for later use
// ----------------------------------------------------------
// Empty the MenuItems table. Using TRUNCATE resets the ID value.
$sql = "TRUNCATE TABLE `" . $this->objectClass . "`";
DB::query($sql);
// This built-in functionality does not reset the ID values.
/*
$q = singleton($this->objectClass)->buildSQL();
$q->select = array('"ID"');
$ids = $q->execute()->column('ID');
foreach($ids as $id) {
$obj = DataObject::get_by_id($this->objectClass, $id);
$obj->delete();
$obj->destroy();
unset($obj);
}
*/
// Set a new instance of this current class
$oThisClass = new $this->objectClass();
// Get the related classes
$aRelatedClasses = $oThisClass->RelatedClasses();
// Loop through the related classes and empty the tables
foreach ($aRelatedClasses as $Class) {
// Empty the related tables. Using TRUNCATE resets the ID values so that the navigation links don't change.
$sql = "TRUNCATE TABLE `" . $Class . "`";
DB::query($sql);
// This built-in functionality does not reset the ID values.
/*
$q = singleton($Class)->buildSQL();
$q->select = array('"ID"');
$ids = $q->execute()->column('ID');
foreach($ids as $id) {
$obj = DataObject::get_by_id($Class, $id);
$obj->delete();
$obj->destroy();
unset($obj);
}
*/
}
}
// Process all the new records
return $this->processAll($filepath);
}
public function publishall($request)
{
if (!Permission::check('ADMIN')) {
return Security::permissionFailure($this);
}
increase_time_limit_to();
increase_memory_limit_to();
$response = "";
if (isset($this->requestParams['confirm'])) {
// Protect against CSRF on destructive action
if (!SecurityToken::inst()->checkRequest($request)) {
return $this->httpError(400);
}
$start = 0;
$pages = DataObject::get("SiteTree", "", "", "", "{$start},30");
$count = 0;
while ($pages) {
foreach ($pages as $page) {
if ($page && !$page->canPublish()) {
return Security::permissionFailure($this);
}
$page->doPublish();
$page->destroy();
unset($page);
$count++;
$response .= "<li>{$count}</li>";
}
if ($pages->Count() > 29) {
$start += 30;
$pages = DataObject::get("SiteTree", "", "", "", "{$start},30");
} else {
break;
}
}
$response .= _t('CMSMain.PUBPAGES', "Done: Published {count} pages", array('count' => $count));
} else {
$token = SecurityToken::inst();
$fields = new FieldList();
$token->updateFieldSet($fields);
$tokenField = $fields->First();
$tokenHtml = $tokenField ? $tokenField->FieldHolder() : '';
$response .= '<h1>' . _t('CMSMain.PUBALLFUN', '"Publish All" functionality') . '</h1>
<p>' . _t('CMSMain.PUBALLFUN2', 'Pressing this button will do the equivalent of going to every page and pressing "publish". It\'s
intended to be used after there have been massive edits of the content, such as when the site was
first built.') . '</p>
<form method="post" action="publishall">
<input type="submit" name="confirm" value="' . _t('CMSMain.PUBALLCONFIRM', "Please publish every page in the site, copying content stage to live", 'Confirmation button') . '" />' . $tokenHtml . '</form>';
}
return $response;
}
public function load($filepath, $memory_limit = '512M')
{
ini_set('max_execution_time', 3600);
increase_memory_limit_to($memory_limit);
return $this->processAll($filepath);
}
