/**
* Get stored remote connections
*
* @param bool $remoteEdit
* @return array
*/
protected function _getRemoteConnections($remoteEdit = true)
{
$remotes = array();
if (!empty($this->_remoteService)) {
$objRFile = new \Components\Projects\Tables\RemoteFile($this->_database);
$remotes = $objRFile->getRemoteFiles($this->model->get('id'), $this->_remoteService, $this->subdir, $remoteEdit);
}
return $remotes;
}
/**
* Fix connection record(s) after local change to parent folder
*
* @param string $service Service name (google)
* @param integer $uid User ID
* @param string $dir Directory path
* @param string $action Action
* @param string $newdir New directory path
* @param string $parentId Parent ID
*
* @return array
*/
public function fixConvertedItems($service = 'google', $uid = 0, $dir = '', $action = '', $newdir = '', $parentId = '')
{
if (!$dir || !$action) {
return false;
}
$objRFile = new \Components\Projects\Tables\RemoteFile($this->_db);
$converted = $objRFile->getRemoteConnections($this->model->get('id'), $service, $dir, 1);
if (!empty($converted['paths'])) {
foreach ($converted['paths'] as $c) {
// Delete record
if ($action == 'D') {
$objRFile->deleteRecord($this->model->get('id'), $service, $c['remote_id']);
} elseif ($newdir) {
// Update dir path
$fpath = $newdir . DS . basename($c['path']);
$update = $objRFile->updateRecord($this->model->get('id'), $service, $c['remote_id'], $fpath, $c['type'], $uid, $parentId);
}
}
}
}
/**
* Sync local and remote changes since last sync
*
* @param string $service Remote service name
* @param boolean $queue Remote service name
* @return void
*/
public function sync($service = 'google', $queue = false, $auto = false)
{
// Lock sync
if (!$this->lockSync($service, false, $queue)) {
// Return error
if ($auto == false) {
$this->setError(Lang::txt('PLG_PROJECTS_FILES_SYNC_DELAYED'));
}
return false;
}
// Record sync status
$this->writeToFile(ucfirst($service) . ' ' . Lang::txt('PLG_PROJECTS_FILES_SYNC_STARTED'));
// Get time of previous sync
$synced = $this->model->params->get($service . '_sync', 1);
// Check for available space
$avail = $this->model->repo()->getAvailableDiskSpace();
// Last synced remote/local change
$lastRemoteChange = $this->model->params->get($service . '_last_remote_change', NULL);
$lastLocalChange = $this->model->params->get($service . '_last_local_change', NULL);
// Get last change ID for project creator
$lastSyncId = $this->model->params->get($service . '_sync_id', NULL);
$prevSyncId = $this->model->params->get($service . '_prev_sync_id', NULL);
// Are we syncing project home directory or other?
$localDir = $this->_connect->getConfigParam($service, 'local_dir');
$localDir = $localDir == '#home' ? '' : $localDir;
$localPath = $this->_path;
$localPath .= $localDir ? DS . $localDir : '';
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_ESTABLISH_REMOTE_CONNECT'));
// User ID of project owner
$projectOwner = $this->model->get('owned_by_user');
// Get service API - always project owner!
$this->_connect->getAPI($service, $projectOwner);
// Collectors
$locals = array();
$remotes = array();
$localRenames = array();
$localFolders = array();
$remoteFolders = array();
$deletes = array();
$timedRemotes = array();
$newRemotes = array();
// Sync start time
$startTime = date('c');
$passed = $synced != 1 ? \Components\Projects\Helpers\Html::timeDifference(strtotime($startTime) - strtotime($synced)) : 'N/A';
// Start debug output
$output = ucfirst($service) . "\n";
$output .= $synced != 1 ? 'Last sync (local): ' . $synced . ' | (UTC): ' . gmdate('Y-m-d H:i:s', strtotime($synced)) . "\n" : "";
$output .= 'Previous sync ID: ' . $prevSyncId . "\n";
$output .= 'Current sync ID: ' . $lastSyncId . "\n";
$output .= 'Last synced remote change: ' . $lastRemoteChange . "\n";
$output .= 'Last synced local change: ' . $lastLocalChange . "\n";
$output .= 'Time passed since last sync: ' . $passed . "\n";
$output .= 'Local sync start time: ' . $startTime . "\n";
$output .= 'Initiated by (user ID): ' . $this->_uid . ' [';
$output .= $auto == true ? 'Auto sync' : 'Manual sync request';
$output .= ']' . "\n";
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_STRUCTURE_REMOTE'));
// Get stored remote connections
$objRFile = new \Components\Projects\Tables\RemoteFile($this->_db);
$connections = $objRFile->getRemoteConnections($this->model->get('id'), $service);
// Get remote folder structure (to find out remote ids)
$this->_connect->getFolderStructure($service, $projectOwner, $remoteFolders);
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_COLLECT_LOCAL'));
$fromLocal = $synced == $lastLocalChange || !$lastLocalChange ? $synced : $lastLocalChange;
// Get all local changes since last sync
$params = array('localPath' => $localPath, 'fromLocal' => $fromLocal, 'localDir' => $localDir, 'localRenames' => $localRenames, 'connections' => $connections);
$locals = $this->model->repo()->getChanges($params);
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_COLLECT_REMOTE'));
// Get all remote files that changed since last sync
$newSyncId = 0;
$nextSyncId = 0;
if ($lastSyncId > 1) {
// Via Changes feed
$newSyncId = $this->_connect->getChangedItems($service, $projectOwner, $lastSyncId, $remotes, $deletes, $connections);
} else {
// Via List feed
$remotes = $this->_connect->getRemoteItems($service, $projectOwner, '', $connections);
$newSyncId = 1;
}
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_VERIFY_REMOTE'));
// Possible that we've missed a change?
if ($newSyncId > $lastSyncId) {
$output .= '!!! Changes detected - new change ID: ' . $newSyncId . "\n";
} else {
$output .= '>>> Returned change ID: ' . $newSyncId . "\n";
}
$output .= empty($remotes) ? 'No changes brought in by Changes feed' . "\n" : 'Changes feed has ' . count($remotes) . ' changes' . "\n";
// Time to get timed remotes from
$from = $synced == $lastRemoteChange || !$lastRemoteChange ? date("c", strtotime($synced) - 1) : date("c", strtotime($lastRemoteChange));
// Get changes via List feed (to make sure we get ALL changes)
// We need this because Changes feed is not 100% reliable :(
if ($newSyncId > $lastSyncId) {
$timedRemotes = $this->_connect->getRemoteItems($service, $projectOwner, $from, $connections);
}
// Record timed remote changes (for debugging)
if (!empty($timedRemotes)) {
$output .= 'Timed remote changes since ' . $from . ' (' . count($timedRemotes) . '):' . "\n";
foreach ($timedRemotes as $tr => $trinfo) {
$output .= $tr . ' changed ' . date("c", $trinfo['time']) . ' status ' . $trinfo['status'] . ' ' . $trinfo['fileSize'] . "\n";
}
// Pick up missed changes
if ($remotes != $timedRemotes) {
$output .= empty($remotes) ? 'Using exclusively timed changes ' . "\n" : 'Mixing in timed changes ' . "\n";
$remotes = $timedRemotes + $remotes;
}
} else {
$output .= 'No timed changes since ' . $from . "\n";
}
// Catch any errors we ran into so far
if ($this->_connect->getError()) {
$this->writeToFile('');
$this->setError(Lang::txt('PLG_PROJECTS_FILES_SYNC_ERROR_OUPS') . ' ' . $this->_connect->getError());
$this->lockSync($service, true);
return false;
}
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_EXPORTING_LOCAL'));
$output .= 'Local changes:' . "\n";
// Go through local changes
if (count($locals) > 0) {
$lChange = NULL;
foreach ($locals as $filename => $local) {
$output .= ' * Local change ' . $filename . ' - ' . $local['status'] . ' - ' . $local['modified'] . ' - ' . $local['time'] . "\n";
// Get latest change
$lChange = $local['time'] > $lChange ? $local['time'] : $lChange;
// Skip renamed files (local renames are handled later)
if (in_array($filename, $localRenames) && !file_exists($local['fullPath'])) {
$output .= '## skipped rename from ' . $filename . "\n";
continue;
}
// Do we have a matching remote change?
$match = !empty($remotes) && isset($remotes[$filename]) && $local['type'] == $remotes[$filename]['type'] ? $remotes[$filename] : NULL;
// Check against individual item sync time (to avoid repeat sync)
if ($local['synced'] && $local['synced'] > $local['modified']) {
$output .= '## item in sync: ' . $filename . ' local: ' . $local['modified'] . ' synced: ' . $local['synced'] . "\n";
continue;
}
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_SYNCING') . ' ' . \Components\Projects\Helpers\Html::shortenFileName($filename, 30));
// Item renamed
if ($local['status'] == 'R') {
if ($local['remoteid']) {
// Rename remote item
$renamed = $this->_connect->renameRemoteItem($this->model->get('id'), $service, $projectOwner, $local['remoteid'], $local, $local['rParent']);
$output .= '>> renamed ' . $local['rename'] . ' to ' . $filename . "\n";
if ($local['type'] == 'folder') {
$this->_connect->fixConvertedItems($service, $this->_uid, $local['rename'], 'R', $filename);
}
continue;
}
}
// Item moved
if ($local['status'] == 'W') {
if ($local['remoteid']) {
// Determine new remote parent
$parentId = $this->_connect->prepRemoteParent($this->model->get('id'), $service, $projectOwner, $local, $remoteFolders);
if ($parentId != $local['rParent']) {
// Move to new parent
$moved = $this->_connect->moveRemoteItem($this->model->get('id'), $service, $projectOwner, $local['remoteid'], $local, $parentId);
$output .= '>> moved ' . $local['rename'] . ' to ' . $filename . ' (new parent id ' . $parentId . ')' . "\n";
if ($local['type'] == 'folder') {
$this->_connect->fixConvertedItems($service, $this->_uid, $local['rename'], 'W', $filename, $parentId);
}
continue;
}
}
}
// Check for match in remote changes
if ($match && $match['time'] - strtotime($from) > 0) {
// skip - remote change prevails
$output .= '== local and remote change match (choosing remote over local): ' . $filename . "\n";
} else {
// Local change needs to be transferred
if ($local['status'] == 'D') {
$deleted = 0;
// Delete operation
if ($local['remoteid']) {
// Delete remote file
$deleted = $this->_connect->deleteRemoteItem($this->model->get('id'), $service, $projectOwner, $local['remoteid'], false);
// Delete from remote
$output .= '-- deleted from remote: ' . $filename . "\n";
} else {
// skip (deleted non-synced file)
$output .= '## skipped deleted non-synced item: ' . $filename . "\n";
$deleted = 1;
}
if ($local['type'] == 'folder') {
$this->_connect->fixConvertedItems($service, $this->_uid, $filename, 'D');
}
// Delete connection record if exists
if ($deleted) {
$objRFile = new \Components\Projects\Tables\RemoteFile($this->_db);
$objRFile->deleteRecord($this->model->get('id'), $service, $local['remoteid'], $filename);
}
} else {
// Not updating converted files via sync
if ($local['converted'] == 1) {
$output .= '## skipped locally changed converted file: ' . $filename . "\n";
} else {
// Item in directory? Make sure we have correct remote dir structure in place
$parentId = $this->_connect->prepRemoteParent($this->model->get('id'), $service, $projectOwner, $local, $remoteFolders);
// Add/update operation
if ($local['remoteid']) {
// Update remote file
$updated = $this->_connect->updateRemoteFile($this->model->get('id'), $service, $projectOwner, $local['remoteid'], $local, $parentId);
$output .= '++ sent update from local to remote: ' . $filename . "\n";
} else {
// Add item from local to remote (new)
if ($local['type'] == 'folder') {
// Create remote folder
$created = $this->_connect->createRemoteFolder($this->model->get('id'), $service, $projectOwner, basename($filename), $filename, $parentId, $remoteFolders);
$output .= '++ created remote folder: ' . $filename . "\n";
} elseif ($local['type'] == 'file') {
// Create remote file
$created = $this->_connect->addRemoteFile($this->model->get('id'), $service, $projectOwner, $local, $parentId);
$output .= '++ added new file to remote: ' . $filename . "\n";
}
}
}
}
}
$lastLocalChange = $lChange ? date('c', $lChange + 1) : NULL;
}
} else {
$output .= 'No local changes since last sync' . "\n";
}
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_REFRESHING_REMOTE'));
// Get new change ID after local changes got sent to remote
if (!empty($locals)) {
$newSyncId = $this->_connect->getChangedItems($service, $projectOwner, $newSyncId, $newRemotes, $deletes, $connections);
}
// Get very last received remote change
if (!empty($remotes)) {
$tChange = strtotime($lastRemoteChange);
foreach ($remotes as $r => $ri) {
$tChange = $ri['time'] > $tChange ? $ri['time'] : $tChange;
}
$lastRemoteChange = $tChange ? date('c', $tChange) : NULL;
}
// Make sure we have thumbnails for updates from local repo
if (!empty($newRemotes) && $synced != 1) {
$tChange = strtotime($lastRemoteChange);
foreach ($newRemotes as $filename => $nR) {
// Generate local thumbnail
if ($nR['thumb']) {
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_GET_THUMB') . ' ' . \Components\Projects\Helpers\Html::shortenFileName($filename, 15));
$this->_connect->generateThumbnail($service, $projectOwner, $nR, $this->model->config(), $this->model->get('alias'));
}
$tChange = $nR['time'] > $tChange ? $nR['time'] : $tChange;
}
// Pick up last remote change
$lastRemoteChange = $tChange ? date('c', $tChange) : NULL;
}
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_IMPORTING_REMOTE'));
$output .= 'Remote changes:' . "\n";
// Go through remote changes
if (count($remotes) > 0 && $synced != 1) {
// Get email/name pairs of connected project owners
$objO = $this->model->table('Owner');
$connected = $objO->getConnected($this->model->get('id'), $service);
// Examine each change
foreach ($remotes as $filename => $remote) {
$output .= ' * Remote change ' . $filename . ' - ' . $remote['status'] . ' - ' . $remote['modified'];
$output .= $remote['fileSize'] ? ' - ' . $remote['fileSize'] . ' bytes' : '';
$output .= "\n";
// Do we have a matching local change?
$match = !empty($locals) && isset($locals[$filename]) && $remote['type'] == $locals[$filename]['type'] ? $locals[$filename] : array();
// Check for match in local changes
// Remote usually prevails, unless it's older than last synced remote change
if ($match && ($match['modified'] > $remote['modified']) > 0) {
// skip
$output .= '== local and remote change match, but remote is older, picking local: ' . $filename . "\n";
continue;
}
// Get change author for Git
$email = '*****@*****.**';
$name = utf8_decode($remote['author']);
if ($connected && isset($connected[$name])) {
$email = $connected[$name];
} else {
// Email from profile?
$email = $objO->getProfileEmail($name, $this->model->get('id'));
}
$author = escapeshellarg($name . ' <' . $email . '> ');
// Change acting user to whoever did the remote change
$uid = $objO->getProfileId($email, $this->model->get('id'));
if ($uid) {
$this->_uid = $uid;
}
$updated = 0;
$deleted = 0;
// Set Git author date (GIT_AUTHOR_DATE)
$cDate = date('c', $remote['time']);
// Important! Needs to be local time, NOT UTC
// Record sync status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_SYNCING') . ' ' . \Components\Projects\Helpers\Html::shortenFileName($filename, 30));
// Item in directory? Make sure we have correct local dir structure
$local_dir = dirname($filename) != '.' ? dirname($filename) : '';
if ($remote['status'] != 'D' && $local_dir && !is_dir($this->_path . DS . $local_dir)) {
// Set params
$params = array('newDir' => $this->_path . DS . $local_dir, 'author' => $author);
$created = $this->model->repo()->makeDirectory($params);
if (!$created) {
$output .= '[error] failed to provision local directory for: ' . $filename . "\n";
continue;
}
}
// Send remote change to local
// Remote version always prevails
if ($remote['status'] == 'D') {
if (file_exists($this->_path . DS . $filename)) {
// Params for repo call
$params = array('item' => $filename, 'type' => $remote['type'], 'author' => $author, 'date' => $cDate);
// Delete local file or directory
if ($deleted = $this->model->repo()->deleteItem($params)) {
$output .= '-- deleted from local: ' . $filename . "\n";
}
} else {
// skip (deleted non-synced file)
$output .= $remote['converted'] == 1 ? '-- deleted converted: ' . $filename . "\n" : '## skipped deleted non-synced item: ' . $filename . "\n";
$deleted = 1;
}
// Delete connection record if exists
if (!empty($deleted)) {
$objRFile = new \Components\Projects\Tables\RemoteFile($this->_db);
$objRFile->deleteRecord($this->model->get('id'), $service, $remote['remoteid']);
}
} elseif ($remote['status'] == 'R' || $remote['status'] == 'W') {
// Rename/move in Git
if (file_exists($this->_path . DS . $remote['rename'])) {
$output .= '>> rename from: ' . $remote['rename'] . ' to ' . $filename . "\n";
// Rename
if ($remote['status'] == 'R') {
// Params for repo call
$params = array('from' => $remote['rename'], 'to' => $filename, 'type' => $remote['type'], 'author' => $author, 'date' => $cDate);
if ($updated = $this->model->repo()->rename($params)) {
$output .= '>> renamed item locally: ' . $filename . "\n";
}
} elseif ($remote['status'] == 'W') {
// Params for repo call
$target = dirname($filename) == '.' ? '' : dirname($filename);
$params = array('item' => $remote['rename'], 'targetDir' => $target, 'type' => $remote['type']);
if ($updated = $this->model->repo()->moveItem($params)) {
$output .= '>> moved item locally: ' . $filename . "\n";
}
}
}
if ($remote['converted'] == 1) {
$output .= '>> renamed/moved item locally converted: ' . $filename . "\n";
$updated = 1;
}
} else {
if ($remote['converted'] == 1) {
// Not updating converted files via sync
$output .= '## skipped converted remotely changed file: ' . $filename . "\n";
$updated = 1;
} elseif (file_exists($this->_path . DS . $filename)) {
// Update
if ($remote['type'] == 'file') {
// Check md5 hash - do we have identical files?
$md5Checksum = hash_file('md5', $this->_path . DS . $filename);
if ($remote['md5'] == $md5Checksum) {
// Skip update
$output .= '## update skipped: local and remote versions identical: ' . $filename . "\n";
$updated = 1;
} else {
// Download remote file
if ($this->_connect->downloadFileCurl($service, $projectOwner, $remote['url'], $this->_path . DS . $remote['local_path'])) {
// Checkin into repo
$this->model->repo()->call('checkin', array('file' => $this->model->repo()->getMetadata($filename, 'file', array()), 'author' => $author, 'date' => $cDate));
$output .= ' ! versions differ: remote md5 ' . $remote['md5'] . ', local md5' . $md5Checksum . "\n";
$output .= '++ sent update from remote to local: ' . $filename . "\n";
$updated = 1;
} else {
// Error
$output .= '[error] failed to update local file with remote change: ' . $filename . "\n";
continue;
}
}
} else {
$output .= '## skipped folder in sync: ' . $filename . "\n";
$updated = 1;
}
} else {
// Add item from remote to local (new)
if ($remote['type'] == 'folder') {
// Set params
$params = array('newDir' => $this->_path . DS . $filename, 'author' => $author);
if ($created = $this->model->repo()->makeDirectory($params)) {
$output .= '++ created local folder: ' . $filename . "\n";
$updated = 1;
} else {
// error
$output .= '[error] failed to create local folder: ' . $filename . "\n";
continue;
}
} else {
// Check against quota
$checkAvail = $avail - $remote['fileSize'];
if ($checkAvail <= 0) {
// Error
$output .= '[error] not enough space for ' . $filename . ' (' . $remote['fileSize'] . ' bytes) avail space:' . $checkAvail . "\n";
continue;
} else {
$avail = $checkAvail;
$output .= 'file size ok: ' . $remote['fileSize'] . ' bytes ' . "\n";
}
// Download remote file
if ($this->_connect->downloadFileCurl($service, $projectOwner, $remote['url'], $this->_path . DS . $remote['local_path'])) {
// Checkin into repo
$this->model->repo()->call('checkin', array('file' => $this->model->repo()->getMetadata($filename, 'file', array()), 'author' => $author, 'date' => $cDate));
$output .= '++ added new file to local: ' . $filename . "\n";
$updated = 1;
} else {
// Error
$output .= '[error] failed to add new local file: ' . $filename . "\n";
continue;
}
}
}
}
// Update connection record
if (!empty($updated)) {
$objRFile = new \Components\Projects\Tables\RemoteFile($this->_db);
$objRFile->updateSyncRecord($this->model->get('id'), $service, $this->_uid, $remote['type'], $remote['remoteid'], $filename, $match, $remote);
$lastLocalChange = date('c', time() + 1);
// Generate local thumbnail
if ($remote['thumb'] && $remote['status'] != 'D') {
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_GET_THUMB') . ' ' . \Components\Projects\Helpers\Html::shortenFileName($filename, 15));
$this->_connect->generateThumbnail($service, $projectOwner, $remote, $this->model->config(), $this->model->get('alias'));
}
}
}
} else {
$output .= 'No remote changes since last sync' . "\n";
}
// Hold on by one second (required as a forced breather before next sync request)
sleep(1);
// Log time
$endTime = date('c');
$length = \Components\Projects\Helpers\Html::timeDifference(strtotime($endTime) - strtotime($startTime));
$output .= 'Sync complete:' . "\n";
$output .= 'Local time: ' . $endTime . "\n";
$output .= 'UTC time: ' . Date::toSql() . "\n";
$output .= 'Sync completed in: ' . $length . "\n";
// Determine next sync ID
if (!$nextSyncId) {
$nextSyncId = $newSyncId > $lastSyncId || count($remotes) > 0 ? $newSyncId + 1 : $lastSyncId;
}
// Save sync time
$this->model->saveParam($service . '_sync', $endTime);
// Save change id for next sync
$this->model->saveParam($service . '_sync_id', $nextSyncId);
$output .= 'Next sync ID: ' . $nextSyncId . "\n";
$this->model->saveParam($service . '_prev_sync_id', $lastSyncId);
$output .= 'Saving last synced remote change at: ' . $lastRemoteChange . "\n";
$this->model->saveParam($service . '_last_remote_change', $lastRemoteChange);
$output .= 'Saving last synced local change at: ' . $lastLocalChange . "\n";
$this->model->saveParam($service . '_last_local_change', $lastLocalChange);
// Debug output
$this->writeToFile($output, $this->_logPath . DS . 'sync.' . Date::of('now')->format('Y-m') . '.log', true);
// Unlock sync
$this->lockSync($service, true);
// Clean up status
$this->writeToFile(Lang::txt('PLG_PROJECTS_FILES_SYNC_COMPLETE'));
$this->set('status', 'success');
return true;
}
