protected function calculateId() { $allChars = '0123456789abcdefghijklmnopqrstuvwxyz'; $dcChars = str_split($allChars, strlen($allChars) / count(kDataCenterMgr::getAllDcs())); $dc = kDataCenterMgr::getCurrentDc(); $dcId = (int) $dc["id"]; $currentDcChars = $dcChars[$dcId]; for ($i = 0; $i < 10; $i++) { $dcChar = substr($currentDcChars, rand(0, strlen($currentDcChars) - 1), 1); if (!$dcChar) { $dcChar = '0'; } $id = $dcChar . kString::generateStringId(3); $existingObject = ShortLinkPeer::retrieveByPK($id); if ($existingObject) { KalturaLog::log("id [{$id}] already exists"); } else { return $id; } } throw new Exception("Could not find unique id for short link"); }
/** * @param EntryDistribution $entryDistribution * @param DistributionProfile $distributionProfile * @param int $dc * @return bool true if the job could be created */ protected static function prepareDistributionJob(EntryDistribution $entryDistribution, DistributionProfile $distributionProfile, &$dc) { // prepare ids list of all the assets $assetIds = explode(',', implode(',', array($entryDistribution->getThumbAssetIds(), $entryDistribution->getFlavorAssetIds()))); $assets = assetPeer::retrieveByIds($assetIds); $assetObjects = array(); foreach ($assets as $asset) { /* @var $asset asset */ $assetObjects[$asset->getId()] = array('asset' => $asset, 'downloadUrl' => null); } // lists all files from all assets $c = new Criteria(); $c->add(FileSyncPeer::OBJECT_TYPE, FileSyncObjectType::ASSET); $c->add(FileSyncPeer::OBJECT_SUB_TYPE, asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET); $c->add(FileSyncPeer::OBJECT_ID, $assetIds, Criteria::IN); $c->add(FileSyncPeer::PARTNER_ID, $entryDistribution->getPartnerId()); $c->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY); $fileSyncs = FileSyncPeer::doSelect($c); $dcs = array(); foreach ($fileSyncs as $fileSync) { /* @var $fileSync FileSync */ $assetId = $fileSync->getObjectId(); if (!isset($assetObjects[$assetId])) { // the object is not in the list of assets continue; } $asset = $assetObjects[$assetId]['asset']; /* @var $asset asset */ if ($asset->getVersion() != $fileSync->getVersion()) { // the file sync is not of the current asset version continue; } $fileSync = kFileSyncUtils::resolve($fileSync); // use the best URL as the source for download in case it will be needed if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) { if (!is_null($assetObjects[$assetId]['downloadUrl']) && $fileSync->getDc() != $distributionProfile->getRecommendedStorageProfileForDownload()) { continue; } $downloadUrl = $fileSync->getExternalUrl($entryDistribution->getEntryId()); if (!$downloadUrl) { continue; } $assetObjects[$assetId]['downloadUrl'] = $downloadUrl; continue; } // populates the list of files in each dc $fileSyncDc = $fileSync->getDc(); if (!isset($dcs[$fileSyncDc])) { $dcs[$fileSyncDc] = array(); } $dcs[$fileSyncDc][$assetId] = $fileSync->getId(); } if (isset($dcs[$dc]) && count($dcs[$dc]) == count($assets)) { KalturaLog::debug("All files exist in the preferred dc [{$dc}]"); return true; } // check if all files exist on any of the remote dcs $otherDcs = kDataCenterMgr::getAllDcs(true); foreach ($otherDcs as $remoteDc) { $remoteDcId = $remoteDc['id']; if (!isset($dcs[$remoteDcId]) && count($dcs[$remoteDcId]) != count($assets)) { continue; } $dc = $remoteDcId; KalturaLog::debug("All files exist in none-preferred dc [{$dc}]"); return true; } if ($entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_SUBMITTING || $entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_UPDATING) { KalturaLog::debug("Entry distribution already importing"); return false; } // create all needed import jobs $destinationDc = $distributionProfile->getRecommendedDcForDownload(); $dcExistingFiles = $dcs[$destinationDc]; foreach ($assetObjects as $assetId => $assetObject) { if (is_null($assetObject['downloadUrl'])) { KalturaLog::debug("Download URL not found for asset [{$assetId}]"); continue; } $asset = $assetObject['asset']; /* @var $asset asset */ if (isset($dcExistingFiles[$assetId])) { continue; } $jobData = new kImportJobData(); $jobData->setCacheOnly(true); self::addImportJob($destinationDc, $assetObject['downloadUrl'], $asset); } return false; }
/** * @param FileSyncKey $key * @param $file_root * @param $real_path * @param $strict * @return SyncFile */ public static function createSyncFileForKey(FileSyncKey $key, $strict = true, $already_exists = false, $cacheOnly = false, $wamsFile = false) { KalturaLog::log(__METHOD__ . " - key [{$key}], strict[{$strict}], already_exists[{$already_exists}]"); // TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING $dc = kDataCenterMgr::getCurrentDc(); $dc_id = $dc["id"]; // create a FileSync for the current DC with status READY if ($already_exists) { $c = FileSyncPeer::getCriteriaForFileSyncKey($key); $c->add(FileSyncPeer::DC, $dc_id); if ($cacheOnly) { $c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_CACHE); } $current_dc_file_sync = FileSyncPeer::doSelectOne($c); } else { list($file_root, $real_path) = kPathManager::getFilePathArr($key); $current_dc_file_sync = FileSync::createForFileSyncKey($key); $current_dc_file_sync->setDc($dc_id); $current_dc_file_sync->setFileRoot($file_root); $current_dc_file_sync->setFilePath($real_path); $current_dc_file_sync->setPartnerId($key->partner_id); $current_dc_file_sync->setOriginal(1); } $full_path = $current_dc_file_sync->getFullPath(); if ($wamsFile) { $current_dc_file_sync->setFileSize(-1); $current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } elseif (file_exists($full_path)) { $current_dc_file_sync->setFileSizeFromPath($full_path); $current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } else { $current_dc_file_sync->setFileSize(-1); if ($strict) { $current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); } else { $current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING); } } if ($cacheOnly) { $current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_CACHE); } else { $current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE); } $current_dc_file_sync->save(); if ($cacheOnly) { return $current_dc_file_sync; } // create records for all other DCs with status PENDING if ($already_exists) { $c = FileSyncPeer::getCriteriaForFileSyncKey($key); $c->add(FileSyncPeer::DC, $dc_id, Criteria::NOT_IN); $remote_dc_file_sync_list = FileSyncPeer::doSelect($c); foreach ($remote_dc_file_sync_list as $remote_dc_file_sync) { $remote_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING); $remote_dc_file_sync->setPartnerID($key->partner_id); $remote_dc_file_sync->save(); } } else { $other_dcs = kDataCenterMgr::getAllDcs(); foreach ($other_dcs as $remote_dc) { $remote_dc_file_sync = FileSync::createForFileSyncKey($key); $remote_dc_file_sync->setDc($remote_dc["id"]); $remote_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING); $remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE); $remote_dc_file_sync->setOriginal(0); $remote_dc_file_sync->setPartnerID($key->partner_id); $remote_dc_file_sync->save(); kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync)); } kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_file_sync)); } return $current_dc_file_sync; }
/** * @param FileSyncKey $key * @param $file_root * @param $real_path * @param $strict * @return SyncFile */ public static function createSyncFileForKey($rootPath, $filePath, FileSyncKey $key, $strict = true, $alreadyExists = false, $cacheOnly = false, $md5 = null) { KalturaLog::debug("key [{$key}], strict[{$strict}], already_exists[{$alreadyExists}]"); // TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING $dc = kDataCenterMgr::getCurrentDc(); $dcId = $dc["id"]; // create a FileSync for the current DC with status READY if ($alreadyExists) { $c = FileSyncPeer::getCriteriaForFileSyncKey($key); $c->add(FileSyncPeer::DC, $dcId); if ($cacheOnly) { $c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_CACHE); } $currentDCFileSync = FileSyncPeer::doSelectOne($c); } else { $currentDCFileSync = FileSync::createForFileSyncKey($key); $currentDCFileSync->setDc($dcId); $currentDCFileSync->setFileRoot($rootPath); $currentDCFileSync->setFilePath($filePath); $currentDCFileSync->setPartnerId($key->partner_id); $currentDCFileSync->setOriginal(1); if (!is_null($md5)) { $currentDCFileSync->setContentMd5($md5); } } $fullPath = $currentDCFileSync->getFullPath(); $isDir = is_dir($fullPath); if (file_exists($fullPath)) { $currentDCFileSync->setFileSizeFromPath($fullPath); $currentDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } else { $currentDCFileSync->setFileSize(-1); if ($strict) { $currentDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); } else { $currentDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING); } } if ($cacheOnly) { $currentDCFileSync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_CACHE); } else { $currentDCFileSync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE); } $currentDCFileSync->setIsDir($isDir); $currentDCFileSync->save(); if ($cacheOnly) { return $currentDCFileSync; } // create records for all other DCs with status PENDING if ($alreadyExists) { $c = FileSyncPeer::getCriteriaForFileSyncKey($key); $c->add(FileSyncPeer::DC, $dcId, Criteria::NOT_IN); $remoteDCFileSyncList = FileSyncPeer::doSelect($c); foreach ($remoteDCFileSyncList as $remoteDCFileSync) { $remoteDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING); $remoteDCFileSync->setPartnerID($key->partner_id); $remoteDCFileSync->setIsDir($isDir); $remoteDCFileSync->save(); } } else { $otherDCs = kDataCenterMgr::getAllDcs(); foreach ($otherDCs as $remoteDC) { $remoteDCFileSync = FileSync::createForFileSyncKey($key); $remoteDCFileSync->setDc($remoteDC["id"]); $remoteDCFileSync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING); $remoteDCFileSync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE); $remoteDCFileSync->setOriginal(0); $remoteDCFileSync->setPartnerID($key->partner_id); $remoteDCFileSync->setIsDir($isDir); $remoteDCFileSync->setFileSize($currentDCFileSync->getFileSize()); $remoteDCFileSync->setOriginalId($currentDCFileSync->getId()); $remoteDCFileSync->setOriginalDc($currentDCFileSync->getDc()); $remoteDCFileSync->save(); kEventsManager::raiseEvent(new kObjectAddedEvent($remoteDCFileSync)); } kEventsManager::raiseEvent(new kObjectAddedEvent($currentDCFileSync)); } return $currentDCFileSync; }
/** * @param FileSyncKey $key * @param $file_root * @param $real_path * @param $strict * @return SyncFile */ public static function createSyncFileLinkForKey(FileSyncKey $target_key, FileSyncKey $source_key, $strict = true) { KalturaLog::log(__METHOD__ . " - target_key [{$target_key}], source_key [{$source_key}]"); // TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING $dc = kDataCenterMgr::getCurrentDc(); $dc_id = $dc["id"]; $sourceFile = self::getLocalFileSyncForKey($source_key, $strict); if (!$sourceFile) { KalturaLog::log(__METHOD__ . " - Warning: no source but NOT strict. target_key [{$target_key}], source_key [{$source_key}] "); return null; } $sourceFile = self::resolve($sourceFile); // we only want to link to a source and not to a link. // create a FileSync for the current DC with status READY $current_dc_file_sync = FileSync::createForFileSyncKey($target_key); $current_dc_file_sync->setDc($dc_id); $current_dc_file_sync->setPartnerId($target_key->partner_id); $current_dc_file_sync->setFileSize(-1); $current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY); $current_dc_file_sync->setReadyAt(time()); $current_dc_file_sync->setOriginal(1); $current_dc_file_sync->setLinkedId($sourceFile->getId()); $current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK); $current_dc_file_sync->save(); //increment link_count for this DC source] self::incrementLinkCountForFileSync($sourceFile); $c = new Criteria(); $c = FileSyncPeer::getCriteriaForFileSyncKey($source_key); $file_sync_list = FileSyncPeer::doSelect($c); $source_file_syncs = array(); foreach ($file_sync_list as $file_sync) { $file_sync = self::resolve($file_sync); // we only want to link to a source and not to a link. $source_file_syncs[$file_sync->getDc()] = $file_sync; } // create records for all other DCs with status READY // link is always ready since no one is going to fetch it. $other_dcs = kDataCenterMgr::getAllDcs(); foreach ($other_dcs as $remote_dc) { // TODO - maybe we should create the file sync if (!isset($source_file_syncs[$remote_dc["id"]])) { continue; } $remote_dc_file_sync = FileSync::createForFileSyncKey($target_key); $remote_dc_file_sync->setDc($remote_dc["id"]); $remote_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY); $remote_dc_file_sync->setOriginal(0); $remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK); $remote_dc_file_sync->setReadyAt(time()); $remote_dc_file_sync->setPartnerID($target_key->partner_id); $remote_dc_file_sync->setLinkedId($source_file_syncs[$remote_dc["id"]]->getId()); $remote_dc_file_sync->save(); // increment link_cont for remote DCs sources self::incrementLinkCountForFileSync($source_file_syncs[$remote_dc["id"]]); kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync)); } kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_file_sync)); }