static function upgrade($version) { log::info("aws_s3", "Commencing module upgrade (" . $version . ")"); switch ($version) { case 0: log::info("aws_s3", "Installing version 1"); @mkdir(VARPATH . "modules/aws_s3"); @mkdir(VARPATH . "modules/aws_s3/log"); // installation's unique identifier - allows multiple g3's pointing to the same s3 bucket. if (!module::get_var("aws_s3", "g3id")) { module::set_var("aws_s3", "g3id", md5(time())); } module::set_var("aws_s3", "synced", false); module::set_var("aws_s3", "enabled", false); module::set_var("aws_s3", "access_key", ""); module::set_var("aws_s3", "secret_key", ""); module::set_var("aws_s3", "bucket_name", ""); module::set_version("aws_s3", 1); case 1: log::info("aws_s3", "Upgrading to version 2"); $db = Database::instance(); $db->query("CREATE TABLE {aws_s3_meta} (\n `item_id` int(9) NOT NULL,\n `item_hash` varchar(32) NOT NULL DEFAULT '',\n `thumb_uploaded` smallint(1) NOT NULL DEFAULT 0,\n `resize_uploaded` smallint(1) NOT NULL DEFAULT 0,\n `fullsize_uploaded` smallint(1) NOT NULL DEFAULT 0,\n `local_deleted` smallint(1) NOT NULL DEFAULT 0,\n PRIMARY KEY (`item_id`)\n ) DEFAULT CHARSET=utf8;"); module::set_var("aws_s3", "upload_thumbs", true); module::set_var("aws_s3", "upload_resizes", true); module::set_var("aws_s3", "upload_fullsizes", true); module::set_var("aws_s3", "s3_storage_only", false); if (module::get_var("aws_s3", "synced")) { // v1 has already synced this installation to s3. mark all the items with the relevant meta data $items = ORM::factory("item")->find_all(); foreach ($items as $item) { aws_s3::log("Updating S3 meta for item ID: " . $item->id); $item->s3_thumb_uploaded = true; if (!$item->is_album()) { $item->s3_resize_uploaded = true; $item->s3_fullsize_uploaded = true; } $item->s3_local_deleted = false; $item->s3_item_hash = md5($item->relative_path()); $item->save_s3_meta(); } } else { // check various states after upgrade from v1.. if (module::get_var("aws_s3", "access_key") != "" && module::get_var("aws_s3", "secret_key") != "" && module::get_var("aws_s3", "bucket_name") != "" && aws_s3::validate_access_details(module::get_var("aws_s3", "access_key"), module::get_var("aws_s3", "secret_key"), module::get_var("aws_s3", "bucket_name"))) { // details are correct but hasn't been synced. if (aws_s3::can_schedule()) { // i can schedule this task aws_s3::schedule_full_sync2(); site_status::warning("Your site has been scheduled for full Amazon S3 re-synchronisation. This message will clear when this has been completed.", "aws_s3_not_synced"); } else { // i CAN'T schedule it.. site_status::warning(t('Your site has not been synchronised to Amazon S3. Until it has, your server will continue to serve image content to your visitors.<br />Click <a href="%url" class="g-dialog-link">here</a> to start the synchronisation task.', array("url" => html::mark_clean(url::site("admin/maintenance/start/aws_s3_task::manual_sync?csrf=__CSRF__")))), "aws_s3_not_synced"); } } else { site_status::warning(t('Amazon S3 module needs configuration. Click <a href="%url">here</a> to go to the configuration page.', array("url" => html::mark_clean(url::site("admin/aws_s3")))), "aws_s3_not_configured"); } } module::set_version("aws_s3", 2); } log::info("aws_s3", "Module upgrade complete"); }
static function remove_album_cover($album) { parent::remove_album_cover($album); if ($album->id > 1) { aws_s3::remove_album_cover($album); } }
public function resize_url($full_uri = false) { if (!module::get_var("aws_s3", "enabled")) { return parent::resize_url($full_uri); } if ($this->is_album() && $this->id > 1) { return aws_s3::generate_url("rs/" . $this->relative_path() . "/.album.jpg", $this->view_1 == 1 ? false : true, $this->updated); } else { return aws_s3::generate_url("rs/" . $this->relative_path(), $this->view_1 == 1 ? false : true, $this->updated); } }
static function item_updated($old_item, $new_item) { if ($new_item->id == 1) { return true; } if ($new_item->has_aws_s3_meta()) { aws_s3::log("Item updated - " . $new_item->id); if ($old_item->relative_path() == $new_item->relative_path() && $old_item->s3_item_hash == $new_item->s3_item_hash) { aws_s3::log("nothing changed?!"); } else { if ($old_item->relative_path() != $new_item->relative_path()) { aws_s3::log("Item moved..."); aws_s3::move_item($old_item, $new_item); } else { aws_s3::log("Item hasn't moved. Image updated?"); aws_s3::remove_item($old_item); aws_s3::schedule_item_sync($new_item); } } } }
static function sync($task) { aws_s3::log("Amazon S3 Re-sync task started.."); batch::start(); $items = ORM::factory("item")->find_all(); $task->set("total_count", count($items)); $task->set("completed", 0); if (!module::get_var("aws_s3", "synced", false)) { aws_s3::log("Emptying contents of bucket"); $task->status = "Emptying contents of bucket"; $task->save(); require_once MODPATH . "aws_s3/lib/s3.php"; $s3 = new S3(module::get_var("aws_s3", "access_key"), module::get_var("aws_s3", "secret_key")); $bucket = module::get_var("aws_s3", "bucket_name"); $resource = aws_s3::get_resource_url(""); $stuff = array_reverse(S3::getBucket($bucket, $resource)); $i = 0; foreach ($stuff as $uri => $item) { $i++; aws_s3::log("Removing " . $uri . " from S3"); S3::deleteObject($bucket, $uri); $task->percent_complete = round(20 * ($i / count($stuff))); $task->save(); } } $task->percent_complete = 20; aws_s3::log("Commencing upload tasks"); $task->state = "Commencing upload..."; $task->save(); $completed = $task->get("completed", 0); $items = ORM::factory("item")->find_all(); foreach ($items as $item) { try { if ($item->id > 1) { aws_s3::upload_item($item, aws_s3::get_upload_flags()); } } catch (Exception $err) { } $completed++; $task->set("completed", $completed); $task->percent_complete = round(80 * ($completed / $task->get("total_count"))) + 20; $task->status = $completed . " of " . $task->get("total_count") . " uploaded."; $task->save(); } $task->percent_complete = 100; $task->state = "success"; $task->done = true; aws_s3::log("Sync task completed successfully"); $task->status = "Sync task completed successfully"; module::set_var("aws_s3", "synced", true); site_status::clear("aws_s3_not_synced"); batch::stop(); $task->save(); }
static function remove_album_cover($album) { self::get_s3(); $success_th = S3::deleteObject(module::get_var("aws_s3", "bucket_name"), "g3/" . module::get_var("aws_s3", "g3id") . "/th/" . $album->relative_path() . "/.album.jpg"); $success_rs = S3::deleteObject(module::get_var("aws_s3", "bucket_name"), "g3/" . module::get_var("aws_s3", "g3id") . "/rs/" . $album->relative_path() . "/.album.jpg"); $success = $success_rs && $success_th; aws_s3::log("album cover removal success: " . $success); }
static function item_moved($new_item, $old_item) { aws_s3::log("Item moved - " . $item->id); aws_s3::move_item($old_item, $new_item); }
static function remove_item($item) { self::get_s3(); $filename = urldecode($item->relative_path()); $itype = "I"; if ($item->is_album()) { $filename .= "/.album.jpg"; $itype = "A"; } if ($item->s3_fullsize_uploaded && !$item->is_album()) { aws_s3::log("[" . $itype . ":" . $item->id . "] Deleting fullsize object"); $success_fs = S3::deleteObject(module::get_var("aws_s3", "bucket_name"), self::get_resource_url("fs/" . $filename)); $item->s3_fullsize_uploaded = !$success_fs; } else { $success_fs = true; } if ($item->s3_resize_uploaded && !$item->is_album()) { aws_s3::log("[" . $itype . ":" . $item->id . "] Deleting resize object"); $success_rs = S3::deleteObject(module::get_var("aws_s3", "bucket_name"), self::get_resource_url("rs/" . $filename)); $item->s3_resize_uploaded = !$success_rs; } else { $success_rs = true; } if ($item->s3_thumb_uploaded) { aws_s3::log("[" . $itype . ":" . $item->id . "] Deleting thumbnail object"); $success_th = S3::deleteObject(module::get_var("aws_s3", "bucket_name"), self::get_resource_url("th/" . $filename)); $item->s3_thumb_uploaded = !$success_th; } else { $success_th = true; } $item->save_s3_meta(); $success = $success_fs && $success_th && $success_rs; aws_s3::log("S3 delete success: " . $success); return $success; }
static function sync($task) { require_once MODPATH . "aws_s3/lib/s3.php"; $s3 = new S3(module::get_var("aws_s3", "access_key"), module::get_var("aws_s3", "secret_key")); $mode = $task->get("mode", "init"); switch ($mode) { case "init": aws_s3::log("re-sync task started.."); batch::start(); $items = ORM::factory("item")->find_all(); aws_s3::log("items to sync: " . count($items)); $task->set("total_count", count($items)); $task->set("completed", 0); $task->set("mode", "empty"); $task->status = "Emptying contents of bucket"; break; case "empty": // 0 - 10% aws_s3::log("emptying bucket contents (any files that may already exist in the bucket/prefix path)"); $bucket = module::get_var("aws_s3", "bucket_name"); $resource = aws_s3::get_resource_url(""); $stuff = array_reverse(S3::getBucket($bucket, $resource)); foreach ($stuff as $uri => $item) { aws_s3::log("removing: " . $uri); S3::deleteObject($bucket, $uri); } $task->percent_complete = 10; $task->set("mode", "upload"); $task->state = "Commencing upload..."; break; case "upload": // 10 - 100% $completed = $task->get("completed", 0); $items = ORM::factory("item")->find_all(1, $completed); foreach ($items as $item) { if ($item->id > 1) { aws_s3::log("uploading item " . $item->id . " (" . ($completed + 1) . "/" . $task->get("total_count") . ")"); if ($item->is_album()) { aws_s3::upload_album_cover($item); } else { aws_s3::upload_item($item); } } $completed++; } $task->set("completed", $completed); $task->percent_complete = round(90 * ($completed / $task->get("total_count"))) + 10; $task->status = $completed . " of " . $task->get("total_count") . " uploaded."; if ($completed == $task->get("total_count")) { $task->set("mode", "finish"); } break; case "finish": aws_s3::log("completing upload task.."); $task->percent_complete = 100; $task->state = "success"; $task->done = true; $task->status = "Sync task completed successfully"; batch::stop(); module::set_var("aws_s3", "synced", true); site_status::clear("aws_s3_not_synced"); break; } }
public function index() { $form = $this->_get_s3_form(); if (request::method() == "post") { access::verify_csrf(); if (($valid_form = $form->validate()) && ($s3_axs_correct = aws_s3::validate_access_details($_POST['access_key'], $_POST['secret_key'], $_POST['bucket_name']))) { // get variable values before changes so we can act on certain changes later $vars = array(); foreach (ORM::factory("var")->where("module_name", "=", "aws_s3")->find_all() as $var) { $vars[$var->name] = $var->value; } // set variables from $_POST into module::set_var() to save module::set_var("aws_s3", "enabled", isset($_POST['enabled']) ? true : false); module::set_var("aws_s3", "access_key", $_POST['access_key']); module::set_var("aws_s3", "secret_key", $_POST['secret_key']); module::set_var("aws_s3", "bucket_name", $_POST['bucket_name']); site_status::clear("aws_s3_not_configured"); module::set_var("aws_s3", "g3id", $_POST['g3id']); module::set_var("aws_s3", "url_str", $_POST['url_str']); module::set_var("aws_s3", "sig_exp", $_POST['sig_exp']); module::set_var("aws_s3", "use_ssl", isset($_POST['use_ssl']) ? true : false); module::set_var("aws_s3", "upload_thumbs", isset($_POST['upload_thumbs']) ? true : false); module::set_var("aws_s3", "upload_resizes", isset($_POST['upload_resizes']) ? true : false); module::set_var("aws_s3", "upload_fullsizes", isset($_POST['upload_fullsizes']) ? true : false); module::set_var("aws_s3", "s3_storage_only", isset($_POST['s3_storage_only']) ? true : false); // post option processing // if (module::get_var("aws_s3", "s3_storage_only") && !module::get_var("aws_s3", "enabled")) { // module::set_var("aws_s3", "enabled", true); // module::set_var("aws_s3", "upload_thumbs", true); // module::set_var("aws_s3", "upload_resizes", true); // module::set_var("aws_s3", "upload_fullsizes", true); // } // if (module::get_var("aws_s3", "s3_storage_only") && !$vars['s3_storage_only']) { // // content needs remove from local storage as it wasn't switched on before this point. // if (!module::get_var("aws_s3", "synced")) { // // force a sync between local storage and S3, as we're about to remove content from local storage. // } // } // else if (!module::get_var("aws_s3", "s3_storage_only") && $vars['s3_storage_only']) { // // content needs to be downloaded from s3 as it was just switched off. at this point, // // we shouldn't actually have a copy of the gallery content locally. // } if (module::get_var("aws_s3", "enabled") && !module::get_var("aws_s3", "synced", false)) { if (aws_s3::can_schedule()) { // i can schedule this task aws_s3::schedule_full_sync2(); site_status::warning("Your site has been scheduled for full Amazon S3 re-synchronisation. This message will clear when this has been completed.", "aws_s3_not_synced"); } else { // i CAN'T schedule it.. site_status::warning(t('Your site has not been synchronised to Amazon S3. Until it has, your server will continue to serve image content to your visitors. Click <a href="%url" class="g-dialog-link">here</a> to start the synchronisation task.', array("url" => html::mark_clean(url::site("admin/maintenance/start/aws_s3_task::manual_sync?csrf=__CSRF__")))), "aws_s3_not_synced"); } } message::success(t("Settings have been saved")); url::redirect("admin/aws_s3"); } else { if (!$valid_form) { message::error(t("There was a problem with the submitted form. Please check your values and try again.")); } if (!$s3_axs_correct) { message::error(t("The Amazon S3 access details provided appear to be incorrect. Please check your values and try again.")); $form->aws_s3->access_key->add_error("invalid", true); $form->aws_s3->secret_key->add_error("invalid", true); $form->aws_s3->bucket_name->add_error("invalid", true); } } } $v = new Admin_View("admin.html"); $v->page_title = t("Amazon S3 Configuration"); $v->content = new View("admin_aws_s3.html"); $v->content->form = $form; $v->content->end = ""; echo $v; }