protected function calculateId()
{
$allChars = '0123456789abcdefghijklmnopqrstuvwxyz';
$dcChars = str_split($allChars, strlen($allChars) / count(kDataCenterMgr::getAllDcs()));
$dc = kDataCenterMgr::getCurrentDc();
$dcId = (int) $dc["id"];
$currentDcChars = $dcChars[$dcId];
for ($i = 0; $i < 10; $i++) {
$dcChar = substr($currentDcChars, rand(0, strlen($currentDcChars) - 1), 1);
if (!$dcChar) {
$dcChar = '0';
}
$id = $dcChar . kString::generateStringId(3);
$existingObject = ShortLinkPeer::retrieveByPK($id);
if ($existingObject) {
KalturaLog::log("id [{$id}] already exists");
} else {
return $id;
}
}
throw new Exception("Could not find unique id for short link");
}
/**
* @param EntryDistribution $entryDistribution
* @param DistributionProfile $distributionProfile
* @param int $dc
* @return bool true if the job could be created
*/
protected static function prepareDistributionJob(EntryDistribution $entryDistribution, DistributionProfile $distributionProfile, &$dc)
{
// prepare ids list of all the assets
$assetIds = explode(',', implode(',', array($entryDistribution->getThumbAssetIds(), $entryDistribution->getFlavorAssetIds())));
$assets = assetPeer::retrieveByIds($assetIds);
$assetObjects = array();
foreach ($assets as $asset) {
/* @var $asset asset */
$assetObjects[$asset->getId()] = array('asset' => $asset, 'downloadUrl' => null);
}
// lists all files from all assets
$c = new Criteria();
$c->add(FileSyncPeer::OBJECT_TYPE, FileSyncObjectType::ASSET);
$c->add(FileSyncPeer::OBJECT_SUB_TYPE, asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET);
$c->add(FileSyncPeer::OBJECT_ID, $assetIds, Criteria::IN);
$c->add(FileSyncPeer::PARTNER_ID, $entryDistribution->getPartnerId());
$c->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY);
$fileSyncs = FileSyncPeer::doSelect($c);
$dcs = array();
foreach ($fileSyncs as $fileSync) {
/* @var $fileSync FileSync */
$assetId = $fileSync->getObjectId();
if (!isset($assetObjects[$assetId])) {
// the object is not in the list of assets
continue;
}
$asset = $assetObjects[$assetId]['asset'];
/* @var $asset asset */
if ($asset->getVersion() != $fileSync->getVersion()) {
// the file sync is not of the current asset version
continue;
}
$fileSync = kFileSyncUtils::resolve($fileSync);
// use the best URL as the source for download in case it will be needed
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) {
if (!is_null($assetObjects[$assetId]['downloadUrl']) && $fileSync->getDc() != $distributionProfile->getRecommendedStorageProfileForDownload()) {
continue;
}
$downloadUrl = $fileSync->getExternalUrl($entryDistribution->getEntryId());
if (!$downloadUrl) {
continue;
}
$assetObjects[$assetId]['downloadUrl'] = $downloadUrl;
continue;
}
// populates the list of files in each dc
$fileSyncDc = $fileSync->getDc();
if (!isset($dcs[$fileSyncDc])) {
$dcs[$fileSyncDc] = array();
}
$dcs[$fileSyncDc][$assetId] = $fileSync->getId();
}
if (isset($dcs[$dc]) && count($dcs[$dc]) == count($assets)) {
KalturaLog::debug("All files exist in the preferred dc [{$dc}]");
return true;
}
// check if all files exist on any of the remote dcs
$otherDcs = kDataCenterMgr::getAllDcs(true);
foreach ($otherDcs as $remoteDc) {
$remoteDcId = $remoteDc['id'];
if (!isset($dcs[$remoteDcId]) && count($dcs[$remoteDcId]) != count($assets)) {
continue;
}
$dc = $remoteDcId;
KalturaLog::debug("All files exist in none-preferred dc [{$dc}]");
return true;
}
if ($entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_SUBMITTING || $entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_UPDATING) {
KalturaLog::debug("Entry distribution already importing");
return false;
}
// create all needed import jobs
$destinationDc = $distributionProfile->getRecommendedDcForDownload();
$dcExistingFiles = $dcs[$destinationDc];
foreach ($assetObjects as $assetId => $assetObject) {
if (is_null($assetObject['downloadUrl'])) {
KalturaLog::debug("Download URL not found for asset [{$assetId}]");
continue;
}
$asset = $assetObject['asset'];
/* @var $asset asset */
if (isset($dcExistingFiles[$assetId])) {
continue;
}
$jobData = new kImportJobData();
$jobData->setCacheOnly(true);
self::addImportJob($destinationDc, $assetObject['downloadUrl'], $asset);
}
return false;
}
/**
* @param FileSyncKey $key
* @param $file_root
* @param $real_path
* @param $strict
* @return SyncFile
*/
public static function createSyncFileForKey(FileSyncKey $key, $strict = true, $already_exists = false, $cacheOnly = false, $wamsFile = false)
{
KalturaLog::log(__METHOD__ . " - key [{$key}], strict[{$strict}], already_exists[{$already_exists}]");
// TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING
$dc = kDataCenterMgr::getCurrentDc();
$dc_id = $dc["id"];
// create a FileSync for the current DC with status READY
if ($already_exists) {
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->add(FileSyncPeer::DC, $dc_id);
if ($cacheOnly) {
$c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_CACHE);
}
$current_dc_file_sync = FileSyncPeer::doSelectOne($c);
} else {
list($file_root, $real_path) = kPathManager::getFilePathArr($key);
$current_dc_file_sync = FileSync::createForFileSyncKey($key);
$current_dc_file_sync->setDc($dc_id);
$current_dc_file_sync->setFileRoot($file_root);
$current_dc_file_sync->setFilePath($real_path);
$current_dc_file_sync->setPartnerId($key->partner_id);
$current_dc_file_sync->setOriginal(1);
}
$full_path = $current_dc_file_sync->getFullPath();
if ($wamsFile) {
$current_dc_file_sync->setFileSize(-1);
$current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
} elseif (file_exists($full_path)) {
$current_dc_file_sync->setFileSizeFromPath($full_path);
$current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
} else {
$current_dc_file_sync->setFileSize(-1);
if ($strict) {
$current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
} else {
$current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING);
}
}
if ($cacheOnly) {
$current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_CACHE);
} else {
$current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE);
}
$current_dc_file_sync->save();
if ($cacheOnly) {
return $current_dc_file_sync;
}
// create records for all other DCs with status PENDING
if ($already_exists) {
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->add(FileSyncPeer::DC, $dc_id, Criteria::NOT_IN);
$remote_dc_file_sync_list = FileSyncPeer::doSelect($c);
foreach ($remote_dc_file_sync_list as $remote_dc_file_sync) {
$remote_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING);
$remote_dc_file_sync->setPartnerID($key->partner_id);
$remote_dc_file_sync->save();
}
} else {
$other_dcs = kDataCenterMgr::getAllDcs();
foreach ($other_dcs as $remote_dc) {
$remote_dc_file_sync = FileSync::createForFileSyncKey($key);
$remote_dc_file_sync->setDc($remote_dc["id"]);
$remote_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING);
$remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE);
$remote_dc_file_sync->setOriginal(0);
$remote_dc_file_sync->setPartnerID($key->partner_id);
$remote_dc_file_sync->save();
kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync));
}
kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_file_sync));
}
return $current_dc_file_sync;
}
/**
* @param FileSyncKey $key
* @param $file_root
* @param $real_path
* @param $strict
* @return SyncFile
*/
public static function createSyncFileForKey($rootPath, $filePath, FileSyncKey $key, $strict = true, $alreadyExists = false, $cacheOnly = false, $md5 = null)
{
KalturaLog::debug("key [{$key}], strict[{$strict}], already_exists[{$alreadyExists}]");
// TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING
$dc = kDataCenterMgr::getCurrentDc();
$dcId = $dc["id"];
// create a FileSync for the current DC with status READY
if ($alreadyExists) {
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->add(FileSyncPeer::DC, $dcId);
if ($cacheOnly) {
$c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_CACHE);
}
$currentDCFileSync = FileSyncPeer::doSelectOne($c);
} else {
$currentDCFileSync = FileSync::createForFileSyncKey($key);
$currentDCFileSync->setDc($dcId);
$currentDCFileSync->setFileRoot($rootPath);
$currentDCFileSync->setFilePath($filePath);
$currentDCFileSync->setPartnerId($key->partner_id);
$currentDCFileSync->setOriginal(1);
if (!is_null($md5)) {
$currentDCFileSync->setContentMd5($md5);
}
}
$fullPath = $currentDCFileSync->getFullPath();
$isDir = is_dir($fullPath);
if (file_exists($fullPath)) {
$currentDCFileSync->setFileSizeFromPath($fullPath);
$currentDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
} else {
$currentDCFileSync->setFileSize(-1);
if ($strict) {
$currentDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
} else {
$currentDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING);
}
}
if ($cacheOnly) {
$currentDCFileSync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_CACHE);
} else {
$currentDCFileSync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE);
}
$currentDCFileSync->setIsDir($isDir);
$currentDCFileSync->save();
if ($cacheOnly) {
return $currentDCFileSync;
}
// create records for all other DCs with status PENDING
if ($alreadyExists) {
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->add(FileSyncPeer::DC, $dcId, Criteria::NOT_IN);
$remoteDCFileSyncList = FileSyncPeer::doSelect($c);
foreach ($remoteDCFileSyncList as $remoteDCFileSync) {
$remoteDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING);
$remoteDCFileSync->setPartnerID($key->partner_id);
$remoteDCFileSync->setIsDir($isDir);
$remoteDCFileSync->save();
}
} else {
$otherDCs = kDataCenterMgr::getAllDcs();
foreach ($otherDCs as $remoteDC) {
$remoteDCFileSync = FileSync::createForFileSyncKey($key);
$remoteDCFileSync->setDc($remoteDC["id"]);
$remoteDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING);
$remoteDCFileSync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE);
$remoteDCFileSync->setOriginal(0);
$remoteDCFileSync->setPartnerID($key->partner_id);
$remoteDCFileSync->setIsDir($isDir);
$remoteDCFileSync->setFileSize($currentDCFileSync->getFileSize());
$remoteDCFileSync->setOriginalId($currentDCFileSync->getId());
$remoteDCFileSync->setOriginalDc($currentDCFileSync->getDc());
$remoteDCFileSync->save();
kEventsManager::raiseEvent(new kObjectAddedEvent($remoteDCFileSync));
}
kEventsManager::raiseEvent(new kObjectAddedEvent($currentDCFileSync));
}
return $currentDCFileSync;
}
/**
* @param FileSyncKey $key
* @param $file_root
* @param $real_path
* @param $strict
* @return SyncFile
*/
public static function createSyncFileLinkForKey(FileSyncKey $target_key, FileSyncKey $source_key, $strict = true)
{
KalturaLog::log(__METHOD__ . " - target_key [{$target_key}], source_key [{$source_key}]");
// TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING
$dc = kDataCenterMgr::getCurrentDc();
$dc_id = $dc["id"];
$sourceFile = self::getLocalFileSyncForKey($source_key, $strict);
if (!$sourceFile) {
KalturaLog::log(__METHOD__ . " - Warning: no source but NOT strict. target_key [{$target_key}], source_key [{$source_key}] ");
return null;
}
$sourceFile = self::resolve($sourceFile);
// we only want to link to a source and not to a link.
// create a FileSync for the current DC with status READY
$current_dc_file_sync = FileSync::createForFileSyncKey($target_key);
$current_dc_file_sync->setDc($dc_id);
$current_dc_file_sync->setPartnerId($target_key->partner_id);
$current_dc_file_sync->setFileSize(-1);
$current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
$current_dc_file_sync->setReadyAt(time());
$current_dc_file_sync->setOriginal(1);
$current_dc_file_sync->setLinkedId($sourceFile->getId());
$current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK);
$current_dc_file_sync->save();
//increment link_count for this DC source]
self::incrementLinkCountForFileSync($sourceFile);
$c = new Criteria();
$c = FileSyncPeer::getCriteriaForFileSyncKey($source_key);
$file_sync_list = FileSyncPeer::doSelect($c);
$source_file_syncs = array();
foreach ($file_sync_list as $file_sync) {
$file_sync = self::resolve($file_sync);
// we only want to link to a source and not to a link.
$source_file_syncs[$file_sync->getDc()] = $file_sync;
}
// create records for all other DCs with status READY
// link is always ready since no one is going to fetch it.
$other_dcs = kDataCenterMgr::getAllDcs();
foreach ($other_dcs as $remote_dc) {
// TODO - maybe we should create the file sync
if (!isset($source_file_syncs[$remote_dc["id"]])) {
continue;
}
$remote_dc_file_sync = FileSync::createForFileSyncKey($target_key);
$remote_dc_file_sync->setDc($remote_dc["id"]);
$remote_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
$remote_dc_file_sync->setOriginal(0);
$remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK);
$remote_dc_file_sync->setReadyAt(time());
$remote_dc_file_sync->setPartnerID($target_key->partner_id);
$remote_dc_file_sync->setLinkedId($source_file_syncs[$remote_dc["id"]]->getId());
$remote_dc_file_sync->save();
// increment link_cont for remote DCs sources
self::incrementLinkCountForFileSync($source_file_syncs[$remote_dc["id"]]);
kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync));
}
kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_file_sync));
}
