/** * Synchronizes a folder to the output stream. Changes for this folders are expected. * * @param SyncCollections $sc * @param SyncParameters $spa * @param IExportChanges $exporter Fully configured exporter for this folder * @param int $changecount Amount of changes expected * @param ImportChangesStream $streamimporter Output stream * @param int $status current status of the folder processing * @param string $newFolderStat the new folder stat to be set if everything was exported * * @throws StatusException * @return int sync status code */ private function syncFolder($sc, $spa, $exporter, $changecount, $streamimporter, $status, $newFolderStat) { $actiondata = $sc->GetParameter($spa, "actiondata"); // send the WBXML start tags (if not happened already) $this->sendFolderStartTag(); self::$encoder->startTag(SYNC_FOLDER); if ($spa->HasContentClass()) { ZLog::Write(LOGLEVEL_DEBUG, sprintf("Folder type: %s", $spa->GetContentClass())); // AS 12.0 devices require content class if (Request::GetProtocolVersion() < 12.1) { self::$encoder->startTag(SYNC_FOLDERTYPE); self::$encoder->content($spa->GetContentClass()); self::$encoder->endTag(); } } self::$encoder->startTag(SYNC_SYNCKEY); if ($status == SYNC_STATUS_SUCCESS && $spa->HasNewSyncKey()) { self::$encoder->content($spa->GetNewSyncKey()); } else { self::$encoder->content($spa->GetSyncKey()); } self::$encoder->endTag(); self::$encoder->startTag(SYNC_FOLDERID); self::$encoder->content($spa->GetFolderId()); self::$encoder->endTag(); self::$encoder->startTag(SYNC_STATUS); self::$encoder->content($status); self::$encoder->endTag(); // announce failing status to the process loop detection if ($status !== SYNC_STATUS_SUCCESS) { self::$deviceManager->AnnounceProcessStatus($spa->GetFolderId(), $status); } // Output IDs and status for incoming items & requests if ($status == SYNC_STATUS_SUCCESS && (!empty($actiondata["clientids"]) || !empty($actiondata["modifyids"]) || !empty($actiondata["removeids"]) || !empty($actiondata["fetchids"]))) { self::$encoder->startTag(SYNC_REPLIES); // output result of all new incoming items foreach ($actiondata["clientids"] as $clientid => $serverid) { self::$encoder->startTag(SYNC_ADD); self::$encoder->startTag(SYNC_CLIENTENTRYID); self::$encoder->content($clientid); self::$encoder->endTag(); if ($serverid) { self::$encoder->startTag(SYNC_SERVERENTRYID); self::$encoder->content($serverid); self::$encoder->endTag(); } self::$encoder->startTag(SYNC_STATUS); self::$encoder->content(isset($actiondata["statusids"][$clientid]) ? $actiondata["statusids"][$clientid] : SYNC_STATUS_CLIENTSERVERCONVERSATIONERROR); self::$encoder->endTag(); self::$encoder->endTag(); } // loop through modify operations which were not a success, send status foreach ($actiondata["modifyids"] as $serverid) { if (isset($actiondata["statusids"][$serverid]) && $actiondata["statusids"][$serverid] !== SYNC_STATUS_SUCCESS) { self::$encoder->startTag(SYNC_MODIFY); self::$encoder->startTag(SYNC_SERVERENTRYID); self::$encoder->content($serverid); self::$encoder->endTag(); self::$encoder->startTag(SYNC_STATUS); self::$encoder->content($actiondata["statusids"][$serverid]); self::$encoder->endTag(); self::$encoder->endTag(); } } // loop through remove operations which were not a success, send status foreach ($actiondata["removeids"] as $serverid) { if (isset($actiondata["statusids"][$serverid]) && $actiondata["statusids"][$serverid] !== SYNC_STATUS_SUCCESS) { self::$encoder->startTag(SYNC_REMOVE); self::$encoder->startTag(SYNC_SERVERENTRYID); self::$encoder->content($serverid); self::$encoder->endTag(); self::$encoder->startTag(SYNC_STATUS); self::$encoder->content($actiondata["statusids"][$serverid]); self::$encoder->endTag(); self::$encoder->endTag(); } } if (!empty($actiondata["fetchids"])) { self::$topCollector->AnnounceInformation(sprintf("Fetching %d objects ", count($actiondata["fetchids"])), $this->singleFolder); $this->saveMultiFolderInfo("fetching", count($actiondata["fetchids"])); } foreach ($actiondata["fetchids"] as $id) { $data = false; try { $fetchstatus = SYNC_STATUS_SUCCESS; // if this is an additional folder the backend has to be setup correctly if (!self::$backend->Setup(ZPush::GetAdditionalSyncFolderStore($spa->GetBackendFolderId()))) { throw new StatusException(sprintf("HandleSync(): could not Setup() the backend to fetch in folder id %s/%s", $spa->GetFolderId(), $spa->GetBackendFolderId()), SYNC_STATUS_OBJECTNOTFOUND); } $data = self::$backend->Fetch($spa->GetBackendFolderId(), $id, $spa->GetCPO()); // check if the message is broken if (ZPush::GetDeviceManager(false) && ZPush::GetDeviceManager()->DoNotStreamMessage($id, $data)) { ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): message not to be streamed as requested by DeviceManager, id = %s", $id)); $fetchstatus = SYNC_STATUS_CLIENTSERVERCONVERSATIONERROR; } } catch (StatusException $stex) { $fetchstatus = $stex->getCode(); } self::$encoder->startTag(SYNC_FETCH); self::$encoder->startTag(SYNC_SERVERENTRYID); self::$encoder->content($id); self::$encoder->endTag(); self::$encoder->startTag(SYNC_STATUS); self::$encoder->content($fetchstatus); self::$encoder->endTag(); if ($data !== false && $status == SYNC_STATUS_SUCCESS) { self::$encoder->startTag(SYNC_DATA); $data->Encode(self::$encoder); self::$encoder->endTag(); } else { ZLog::Write(LOGLEVEL_WARN, sprintf("Unable to Fetch '%s'", $id)); } self::$encoder->endTag(); } self::$encoder->endTag(); } if ($sc->GetParameter($spa, "getchanges") && $spa->HasFolderId() && $spa->HasContentClass() && $spa->HasSyncKey()) { $moreAvailableSent = false; $windowSize = self::$deviceManager->GetWindowSize($spa->GetFolderId(), $spa->GetUuid(), $spa->GetUuidCounter(), $changecount); // limit windowSize to the max available limit of the global window size left $globallyAvailable = $sc->GetGlobalWindowSize() - $this->globallyExportedItems; if ($changecount > $globallyAvailable && $windowSize > $globallyAvailable) { ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): Limit window size to %d as the global window size limit will be reached", $globallyAvailable)); $windowSize = $globallyAvailable; } // send <MoreAvailable/> if there are more changes than fit in the folder windowsize // or there is a move state (another sync should be done afterwards) if ($changecount > $windowSize || $spa->GetMoveState() !== false) { self::$encoder->startTag(SYNC_MOREAVAILABLE, false, true); $moreAvailableSent = true; $spa->DelFolderStat(); } } // Stream outgoing changes if ($status == SYNC_STATUS_SUCCESS && $sc->GetParameter($spa, "getchanges") == true && $windowSize > 0 && !!$exporter) { self::$topCollector->AnnounceInformation(sprintf("Streaming data of %d objects", $changecount > $windowSize ? $windowSize : $changecount)); // Output message changes per folder self::$encoder->startTag(SYNC_PERFORM); $n = 0; WBXMLDecoder::ResetInWhile("syncSynchronize"); while (WBXMLDecoder::InWhile("syncSynchronize")) { try { $progress = $exporter->Synchronize(); if (!is_array($progress)) { break; } $n++; if ($n % 10 == 0) { self::$topCollector->AnnounceInformation(sprintf("Streamed data of %d objects out of %d", $n, $changecount > $windowSize ? $windowSize : $changecount)); } } catch (SyncObjectBrokenException $mbe) { $brokenSO = $mbe->GetSyncObject(); if (!$brokenSO) { ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): Catched SyncObjectBrokenException but broken SyncObject not available. This should be fixed in the backend.")); } else { if (!isset($brokenSO->id)) { $brokenSO->id = "Unknown ID"; ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): Catched SyncObjectBrokenException but no ID of object set. This should be fixed in the backend.")); } self::$deviceManager->AnnounceIgnoredMessage($spa->GetFolderId(), $brokenSO->id, $brokenSO); } } catch (StatusException $stex) { $status = $stex->getCode(); // during export we found out that the states should be thrown away (ZP-623) if ($status == SYNC_STATUS_INVALIDSYNCKEY) { self::$deviceManager->ForceFolderResync($spa->GetFolderId()); break; } } if ($n >= $windowSize || Request::IsRequestTimeoutReached()) { ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): Exported maxItems of messages: %d / %d", $n, $changecount)); break; } } // $progress is not an array when exporting the last message // so we get the number to display from the streamimporter if it's available if (!!$streamimporter) { $n = $streamimporter->GetImportedMessages(); } self::$encoder->endTag(); // log the request timeout if (Request::IsRequestTimeoutReached()) { ZLog::Write(LOGLEVEL_DEBUG, "HandleSync(): Stopping export as maximum request timeout is almost reached!"); // Send a <MoreAvailable/> tag if we reached the request timeout, there are more changes and a moreavailable was not already send if (!$moreAvailableSent && $n > $windowSize) { self::$encoder->startTag(SYNC_MOREAVAILABLE, false, true); $spa->DelFolderStat(); $moreAvailableSent = true; } } self::$topCollector->AnnounceInformation(sprintf("Outgoing %d objects%s", $n, $n >= $windowSize ? " of " . $changecount : ""), $this->singleFolder); $this->saveMultiFolderInfo("outgoing", $n); $this->saveMultiFolderInfo("queued", $changecount); $this->globallyExportedItems += $n; // update folder status, if there is something set if ($spa->GetFolderSyncRemaining() && $changecount > 0) { $spa->SetFolderSyncRemaining($changecount); } // changecount is initialized with 'false', so 0 means no changes! if ($changecount === 0 || $changecount !== false && $changecount <= $windowSize) { self::$deviceManager->SetFolderSyncStatus($spa->GetFolderId(), DeviceManager::FLD_SYNC_COMPLETED); // we should update the folderstat, but we recheck to see if it changed. If so, it's not updated to force another sync $newFolderStatAfterExport = self::$backend->GetFolderStat(ZPush::GetAdditionalSyncFolderStore($spa->GetBackendFolderId()), $spa->GetBackendFolderId()); if ($newFolderStat === $newFolderStatAfterExport) { $this->setFolderStat($spa, $newFolderStat); } else { ZLog::Write(LOGLEVEL_DEBUG, "Sync() Folderstat differs after export, force another exporter run."); } } else { self::$deviceManager->SetFolderSyncStatus($spa->GetFolderId(), DeviceManager::FLD_SYNC_INPROGRESS); } } self::$encoder->endTag(); // Save the sync state for the next time if ($spa->HasNewSyncKey()) { self::$topCollector->AnnounceInformation("Saving state"); try { if (isset($exporter) && $exporter) { $state = $exporter->GetState(); // update the move state (it should be gone now) list($moveState, ) = $exporter->GetMoveStates(); $spa->SetMoveState($moveState); } else { if ($sc->GetParameter($spa, "state") !== null) { $state = $sc->GetParameter($spa, "state"); } else { if (!$spa->HasSyncKey()) { $state = ""; } } } } catch (StatusException $stex) { $status = $stex->getCode(); } if (isset($state) && $status == SYNC_STATUS_SUCCESS) { self::$deviceManager->GetStateManager()->SetSyncState($spa->GetNewSyncKey(), $state, $spa->GetFolderId()); } else { ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): error saving '%s' - no state information available", $spa->GetNewSyncKey())); } } // save SyncParameters if ($status == SYNC_STATUS_SUCCESS && empty($actiondata["fetchids"])) { $sc->SaveCollection($spa); } return $status; }
/** * Handles the FolderSync command * * @param int $commandCode * * @access public * @return boolean */ public function Handle($commandCode) { // Parse input if (!self::$decoder->getElementStartTag(SYNC_FOLDERHIERARCHY_FOLDERSYNC)) { return false; } if (!self::$decoder->getElementStartTag(SYNC_FOLDERHIERARCHY_SYNCKEY)) { return false; } $synckey = self::$decoder->getElementContent(); if (!self::$decoder->getElementEndTag()) { return false; } // every FolderSync with SyncKey 0 should return the supported AS version & command headers if ($synckey == "0") { self::$specialHeaders = array(); self::$specialHeaders[] = ZPush::GetSupportedProtocolVersions(); self::$specialHeaders[] = ZPush::GetSupportedCommands(); } $status = SYNC_FSSTATUS_SUCCESS; $newsynckey = $synckey; try { $syncstate = self::$deviceManager->GetStateManager()->GetSyncState($synckey); // We will be saving the sync state under 'newsynckey' $newsynckey = self::$deviceManager->GetStateManager()->GetNewSyncKey($synckey); // there are no SyncParameters for the hierarchy, but we use it to save the latest synckeys $spa = self::$deviceManager->GetStateManager()->GetSynchedFolderState(false); } catch (StateNotFoundException $snfex) { $status = SYNC_FSSTATUS_SYNCKEYERROR; } catch (StateInvalidException $sive) { $status = SYNC_FSSTATUS_SYNCKEYERROR; } // The ChangesWrapper caches all imports in-memory, so we can send a change count // before sending the actual data. // the HierarchyCache is notified and the changes from the PIM are transmitted to the actual backend $changesMem = self::$deviceManager->GetHierarchyChangesWrapper(); // the hierarchyCache should now fully be initialized - check for changes in the additional folders $changesMem->Config(ZPush::GetAdditionalSyncFolders(false)); // reset to default store in backend self::$backend->Setup(false); // process incoming changes if (self::$decoder->getElementStartTag(SYNC_FOLDERHIERARCHY_CHANGES)) { // Ignore <Count> if present if (self::$decoder->getElementStartTag(SYNC_FOLDERHIERARCHY_COUNT)) { self::$decoder->getElementContent(); if (!self::$decoder->getElementEndTag()) { return false; } } // Process the changes (either <Add>, <Modify>, or <Remove>) $element = self::$decoder->getElement(); if ($element[EN_TYPE] != EN_TYPE_STARTTAG) { return false; } $importer = false; WBXMLDecoder::ResetInWhile("folderSyncIncomingChange"); while (WBXMLDecoder::InWhile("folderSyncIncomingChange")) { $folder = new SyncFolder(); if (!$folder->Decode(self::$decoder)) { break; } // add the backendId to the SyncFolder object $folder->BackendId = self::$deviceManager->GetBackendIdForFolderId($folder->serverid); try { if ($status == SYNC_FSSTATUS_SUCCESS && !$importer) { // Configure the backends importer with last state $importer = self::$backend->GetImporter(); $importer->Config($syncstate); // the messages from the PIM will be forwarded to the backend $changesMem->forwardImporter($importer); } if ($status == SYNC_FSSTATUS_SUCCESS) { switch ($element[EN_TAG]) { case SYNC_ADD: case SYNC_MODIFY: $serverid = $changesMem->ImportFolderChange($folder); break; case SYNC_REMOVE: $serverid = $changesMem->ImportFolderDeletion($folder); break; } } else { ZLog::Write(LOGLEVEL_WARN, sprintf("Request->HandleFolderSync(): ignoring incoming folderchange for folder '%s' as status indicates problem.", $folder->displayname)); self::$topCollector->AnnounceInformation("Incoming change ignored", true); } } catch (StatusException $stex) { $status = $stex->getCode(); } } if (!self::$decoder->getElementEndTag()) { return false; } } else { // check for a potential process loop like described in Issue ZP-5 if ($synckey != "0" && self::$deviceManager->IsHierarchyFullResyncRequired()) { $status = SYNC_FSSTATUS_SYNCKEYERROR; } self::$deviceManager->AnnounceProcessStatus(false, $status); } if (!self::$decoder->getElementEndTag()) { return false; } // We have processed incoming foldersync requests, now send the PIM // our changes // Output our WBXML reply now self::$encoder->StartWBXML(); self::$encoder->startTag(SYNC_FOLDERHIERARCHY_FOLDERSYNC); if ($status == SYNC_FSSTATUS_SUCCESS) { try { // do nothing if this is an invalid device id (like the 'validate' Androids internal client sends) if (!Request::IsValidDeviceID()) { throw new StatusException(sprintf("Request::IsValidDeviceID() indicated that '%s' is not a valid device id", Request::GetDeviceID()), SYNC_FSSTATUS_SERVERERROR); } // Changes from backend are sent to the MemImporter and processed for the HierarchyCache. // The state which is saved is from the backend, as the MemImporter is only a proxy. $exporter = self::$backend->GetExporter(); $exporter->Config($syncstate); $exporter->InitializeExporter($changesMem); // Stream all changes to the ImportExportChangesMem $totalChanges = $exporter->GetChangeCount(); $exported = 0; $partial = false; while (is_array($exporter->Synchronize())) { $exported++; if (time() % 4) { self::$topCollector->AnnounceInformation(sprintf("Exported %d from %d folders", $exported, $totalChanges)); } // if partial sync is allowed, stop if this takes too long if (USE_PARTIAL_FOLDERSYNC && Request::IsRequestTimeoutReached()) { ZLog::Write(LOGLEVEL_WARN, sprintf("Request->HandleFolderSync(): Exporting folders is too slow. In %d seconds only %d from %d changes were processed.", time() - $_SERVER["REQUEST_TIME"], $exported, $totalChanges)); self::$topCollector->AnnounceInformation(sprintf("Partial export of %d out of %d folders", $exported, $totalChanges), true); self::$deviceManager->SetFolderSyncComplete(false); $partial = true; break; } } // update the foldersync complete flag if (USE_PARTIAL_FOLDERSYNC && $partial == false && self::$deviceManager->GetFolderSyncComplete() === false) { // say that we are done with partial synching self::$deviceManager->SetFolderSyncComplete(true); // reset the loop data to prevent any loop detection to kick in now self::$deviceManager->ClearLoopDetectionData(Request::GetAuthUser(), Request::GetDeviceID()); ZLog::Write(LOGLEVEL_INFO, "Request->HandleFolderSync(): Chunked exporting of folders completed successfully"); } // get the new state from the backend $newsyncstate = isset($exporter) ? $exporter->GetState() : ""; } catch (StatusException $stex) { if ($stex->getCode() == SYNC_FSSTATUS_CODEUNKNOWN) { $status = SYNC_FSSTATUS_SYNCKEYERROR; } else { $status = $stex->getCode(); } } } self::$encoder->startTag(SYNC_FOLDERHIERARCHY_STATUS); self::$encoder->content($status); self::$encoder->endTag(); if ($status == SYNC_FSSTATUS_SUCCESS) { self::$encoder->startTag(SYNC_FOLDERHIERARCHY_SYNCKEY); $synckey = $changesMem->IsStateChanged() ? $newsynckey : $synckey; self::$encoder->content($synckey); self::$encoder->endTag(); // Stream folders directly to the PDA $streamimporter = new ImportChangesStream(self::$encoder, false); $changesMem->InitializeExporter($streamimporter); $changeCount = $changesMem->GetChangeCount(); self::$encoder->startTag(SYNC_FOLDERHIERARCHY_CHANGES); self::$encoder->startTag(SYNC_FOLDERHIERARCHY_COUNT); self::$encoder->content($changeCount); self::$encoder->endTag(); while ($changesMem->Synchronize()) { } self::$encoder->endTag(); self::$topCollector->AnnounceInformation(sprintf("Outgoing %d folders", $changeCount), true); // everything fine, save the sync state for the next time if ($synckey == $newsynckey) { self::$deviceManager->GetStateManager()->SetSyncState($newsynckey, $newsyncstate); // update SPA & save it $spa->SetSyncKey($newsynckey); $spa->SetFolderId(false); self::$deviceManager->GetStateManager()->SetSynchedFolderState($spa); // invalidate all pingable flags SyncCollections::InvalidatePingableFlags(); } } self::$encoder->endTag(); return true; }