/**
* Loads emails from an email server and save them into the database
*
* @param EmailFolder $folder
* @param SearchQuery $searchQuery
*/
protected function loadEmails(EmailFolder $folder, SearchQuery $searchQuery)
{
$this->log->notice(sprintf('Query: "%s".', $searchQuery->convertToSearchString()));
$folder->setSynchronizedAt(new \DateTime('now', new \DateTimeZone('UTC')));
$emails = $this->manager->getEmails($searchQuery);
$needFolderFlush = true;
$count = 0;
$batch = array();
foreach ($emails as $email) {
$count++;
$batch[] = $email;
if ($count === self::DB_BATCH_SIZE) {
$this->saveEmails($batch, $folder);
$needFolderFlush = false;
$count = 0;
$batch = array();
}
}
if ($count > 0) {
$this->saveEmails($batch, $folder);
$needFolderFlush = false;
}
if ($needFolderFlush) {
$this->em->flush();
}
}
/**
* @param Message[] $batch
*/
protected function handleBatchLoaded($batch)
{
$this->batch = [];
foreach ($batch as $key => $val) {
$this->batch[$key] = $this->manager->convertToEmail($val);
}
}
public function testConvertToEmailWithMultiValueAcceptLanguage()
{
$msg = $this->getMockBuilder('Oro\\Bundle\\ImapBundle\\Mail\\Storage\\Message')->disableOriginalConstructor()->getMock();
$headers = $this->getMockBuilder('Zend\\Mail\\Headers')->disableOriginalConstructor()->getMock();
$msg->expects($this->once())->method('getHeaders')->will($this->returnValue($headers));
$headers->expects($this->any())->method('get')->will($this->returnValueMap([['UID', $this->getHeader('123')], ['Subject', $this->getHeader('Subject')], ['From', $this->getHeader('fromEmail')], ['Date', $this->getHeader('Fri, 31 Jun 2011 10:59:59 +1100')], ['Received', $this->getHeader('by server to email; Fri, 31 Jun 2011 10:58:58 +1100')], ['InternalDate', $this->getHeader('Fri, 31 Jun 2011 10:57:57 +1100')], ['Message-ID', $this->getHeader('MessageId')], ['Importance', false], ['References', $this->getHeader('References')], ['X-GM-MSG-ID', $this->getHeader('XMsgId')], ['X-GM-THR-ID', $this->getHeader('XThrId1')], ['X-GM-LABELS', false], ['Accept-Language', $this->getMultiValueHeader(['en-US', 'en-US'])]]));
$email = $this->manager->convertToEmail($msg);
$this->assertNotEmpty($email->getMessageId());
$this->assertEquals('en-US', $email->getAcceptLanguageHeader());
}
/**
* {@inheritdoc}
*/
public function loadEmailBody(EmailFolder $folder, Email $email, EntityManager $em)
{
/** @var UserEmailOrigin $origin */
$origin = $folder->getOrigin();
$config = new ImapConfig($origin->getImapHost(), $origin->getImapPort(), $origin->getImapEncryption(), $origin->getUser(), $this->encryptor->decryptData($origin->getPassword()), $this->imapEmailGoogleOauth2Manager->getAccessTokenWithCheckingExpiration($origin));
$manager = new ImapEmailManager($this->connectorFactory->createImapConnector($config));
$manager->selectFolder($folder->getFullName());
$repo = $em->getRepository('OroImapBundle:ImapEmail');
$query = $repo->createQueryBuilder('e')->select('e.uid')->innerJoin('e.imapFolder', 'if')->where('e.email = ?1 AND if.folder = ?2')->setParameter(1, $email)->setParameter(2, $folder)->getQuery();
$loadedEmail = $manager->findEmail($query->getSingleScalarResult());
if (null === $loadedEmail) {
throw new EmailBodyNotFoundException($email);
}
$builder = new EmailBodyBuilder();
$builder->setEmailBody($loadedEmail->getBody()->getContent(), $loadedEmail->getBody()->getBodyIsText());
foreach ($loadedEmail->getAttachments() as $attachment) {
$builder->addEmailAttachment($attachment->getFileName(), $attachment->getContent(), $attachment->getContentType(), $attachment->getContentTransferEncoding(), $attachment->getContentId());
}
return $builder->getEmailBody();
}
/**
* {@inheritdoc}
*/
public function loadEmailBody(Email $email, EntityManager $em)
{
/** @var ImapEmailOrigin $origin */
$origin = $email->getFolder()->getOrigin();
$config = new ImapConfig($origin->getHost(), $origin->getPort(), $origin->getSsl(), $origin->getUser(), $this->encryptor->decryptData($origin->getPassword()));
$manager = new ImapEmailManager($this->connectorFactory->createImapConnector($config));
$manager->selectFolder($email->getFolder()->getFullName());
$repo = $em->getRepository('OroImapBundle:ImapEmail');
$query = $repo->createQueryBuilder('e')->select('e.uid')->where('e.email = ?1')->setParameter(1, $email)->getQuery();
/** @var ImapEmail $imapEmail */
$imapEmail = $query->getSingleResult();
$loadedEmail = $manager->findEmail($imapEmail['uid']);
if ($loadedEmail === null) {
throw new \RuntimeException(sprintf('Cannot find a body for "%s" email.', $email->getSubject()));
}
$builder = new EmailBodyBuilder();
$builder->setEmailBody($loadedEmail->getBody()->getContent(), $loadedEmail->getBody()->getBodyIsText());
foreach ($loadedEmail->getAttachments() as $attachment) {
$builder->addEmailAttachment($attachment->getFileName(), $attachment->getContent(), $attachment->getContentType(), $attachment->getContentTransferEncoding());
}
return $builder->getEmailBody();
}
/**
* Saves emails into the database
*
* @param Email[] $emails
* @param ImapEmailFolder $imapFolder
*/
protected function saveEmails(array $emails, ImapEmailFolder $imapFolder)
{
$this->emailEntityBuilder->removeEmails();
$folder = $imapFolder->getFolder();
$existingUids = $this->getExistingUids($folder, $emails);
$isMultiFolder = $this->manager->hasCapability(Imap::CAPABILITY_MSG_MULTI_FOLDERS);
$messageIds = $this->getNewMessageIds($emails, $existingUids);
$existingImapEmails = $this->getExistingImapEmails($folder->getOrigin(), $messageIds, $isMultiFolder);
$existingEmailUsers = $this->getExistingEmailUsers($folder, $messageIds);
/** @var ImapEmail[] $newImapEmails */
$newImapEmails = [];
foreach ($emails as $email) {
if (!$this->allowSaveEmail($folder, $email, $existingUids)) {
continue;
}
/** @var ImapEmail[] $relatedExistingImapEmails */
$relatedExistingImapEmails = array_filter($existingImapEmails, function (ImapEmail $imapEmail) use($email) {
return $imapEmail->getEmail()->getMessageId() === $email->getMessageId();
});
$existingImapEmail = $this->findExistingImapEmail($relatedExistingImapEmails, $folder->getType(), $isMultiFolder);
if ($existingImapEmail) {
$this->moveEmailToOtherFolder($existingImapEmail, $imapFolder, $email->getId()->getUid());
} else {
try {
$emailUser = isset($existingEmailUsers[$email->getMessageId()]) ? $existingEmailUsers[$email->getMessageId()] : $this->addEmailUser($email, $folder, $email->hasFlag("\\Seen"), $this->currentUser, $this->currentOrganization);
$imapEmail = $this->createImapEmail($email->getId()->getUid(), $emailUser->getEmail(), $imapFolder);
$newImapEmails[] = $imapEmail;
$this->em->persist($imapEmail);
$this->logger->notice(sprintf('The "%s" (UID: %d) email was persisted.', $email->getSubject(), $email->getId()->getUid()));
} catch (\Exception $e) {
$this->logger->warning(sprintf('Failed to persist "%s" (UID: %d) email. Error: %s', $email->getSubject(), $email->getId()->getUid(), $e->getMessage()));
}
}
$this->removeEmailFromOutdatedFolders($relatedExistingImapEmails);
}
$this->emailEntityBuilder->getBatch()->persist($this->em);
// update references if needed
$changes = $this->emailEntityBuilder->getBatch()->getChanges();
foreach ($newImapEmails as $imapEmail) {
foreach ($changes as $change) {
if ($change['old'] instanceof EmailEntity && $imapEmail->getEmail() === $change['old']) {
$imapEmail->setEmail($change['new']);
}
}
}
$this->em->flush();
$this->cleanUp();
}
/**
* Saves emails into the database
*
* @param Email[] $emails
* @param ImapEmailFolder $imapFolder
*/
protected function saveEmails(array $emails, ImapEmailFolder $imapFolder)
{
$this->emailEntityBuilder->removeEmails();
$folder = $imapFolder->getFolder();
$existingUids = $this->getExistingUids($folder, $emails);
$isMultiFolder = $this->manager->hasCapability(Imap::CAPABILITY_MSG_MULTI_FOLDERS);
$existingImapEmails = $this->getExistingImapEmails($folder->getOrigin(), $this->getNewMessageIds($emails, $existingUids), $isMultiFolder);
/** @var ImapEmail[] $newImapEmails */
$newImapEmails = [];
foreach ($emails as $email) {
if (in_array($email->getId()->getUid(), $existingUids)) {
$this->log->notice(sprintf('Skip "%s" (UID: %d) email, because it is already synchronised.', $email->getSubject(), $email->getId()->getUid()));
continue;
}
/** @var ImapEmail[] $relatedExistingImapEmails */
$relatedExistingImapEmails = array_filter($existingImapEmails, function (ImapEmail $imapEmail) use($email) {
return $imapEmail->getEmail()->getMessageId() === $email->getMessageId();
});
$existingImapEmail = $this->findExistingImapEmail($relatedExistingImapEmails, $folder->getType(), $isMultiFolder);
if ($existingImapEmail) {
$this->moveEmailToOtherFolder($existingImapEmail, $imapFolder, $email->getId()->getUid());
} else {
$this->log->notice(sprintf('Persisting "%s" email (UID: %d) ...', $email->getSubject(), $email->getId()->getUid()));
$imapEmail = $this->createImapEmail($email->getId()->getUid(), $this->addEmail($email, $folder), $imapFolder);
$newImapEmails[] = $imapEmail;
$this->em->persist($imapEmail);
$this->log->notice(sprintf('The "%s" email was persisted.', $email->getSubject()));
}
$this->removeEmailFromOutdatedFolders($relatedExistingImapEmails);
}
$this->emailEntityBuilder->getBatch()->persist($this->em);
// update references if needed
$changes = $this->emailEntityBuilder->getBatch()->getChanges();
foreach ($newImapEmails as $imapEmail) {
foreach ($changes as $change) {
if ($change['old'] instanceof EmailEntity && $imapEmail->getEmail() === $change['old']) {
$imapEmail->setEmail($change['new']);
}
}
}
$this->em->flush();
}
/**
* @param Message[] $batch
*
* @throws \Exception
*/
protected function handleBatchLoaded($batch)
{
$this->batch = [];
$counter = 0;
foreach ($batch as $key => $val) {
try {
$email = $this->manager->convertToEmail($val);
} catch (\Exception $e) {
if (null !== $this->onConvertError) {
call_user_func($this->onConvertError, $e);
$email = null;
} else {
throw $e;
}
}
// do not add invalid messages to $this->batch
if ($email !== null) {
$this->batch[$this->iterationPos + $counter] = $email;
}
$counter++;
}
}
/**
* Get email ids and create iterator
*
* @param EmailOrigin $origin
* @param ImapEmailFolder $imapFolder
* @param EmailFolder $folder
*
* @return ImapEmailIterator
*/
protected function getEmailIterator(EmailOrigin $origin, ImapEmailFolder $imapFolder, EmailFolder $folder)
{
if ($origin->getMailbox()) {
// build search query for emails sync
$sqb = $this->manager->getSearchQueryBuilder();
if ($origin->getSynchronizedAt() && $folder->getSynchronizedAt()) {
if ($folder->getType() === FolderType::SENT) {
$sqb->sent($folder->getSynchronizedAt());
} else {
$sqb->received($folder->getSynchronizedAt());
}
}
$searchQuery = $sqb->get();
$this->logger->info(sprintf('Loading emails from "%s" folder ...', $folder->getFullName()));
$this->logger->info(sprintf('Query: "%s".', $searchQuery->convertToSearchString()));
$emails = $this->manager->getEmails($searchQuery);
} else {
$lastUid = $this->em->getRepository('OroImapBundle:ImapEmail')->findLastUidByFolder($imapFolder);
$this->logger->info(sprintf('Previous max email UID "%s"', $lastUid));
$emails = $this->manager->getEmailsUidBased(sprintf('%s:*', ++$lastUid));
}
return $emails;
}
/**
* Return the current element
*
* @return Email
*/
public function current()
{
return $this->manager->convertToEmail($this->iterator->current());
}
/**
* @SuppressWarnings(PHPMD.ExcessiveMethodLength)
*/
public function testGetEmails()
{
$uid = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$uid->expects($this->once())->method('getFieldValue')->will($this->returnValue('123'));
$subject = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$subject->expects($this->once())->method('getFieldValue')->will($this->returnValue('Subject'));
$fromEmail = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$fromEmail->expects($this->once())->method('getFieldValue')->will($this->returnValue('fromEmail'));
$date = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$date->expects($this->once())->method('getFieldValue')->will($this->returnValue('2011-06-30 23:59:59 +0'));
$received = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$received->expects($this->once())->method('getFieldValue')->will($this->returnValue('by server to email; 2012-06-30 23:59:59 +0'));
$intDate = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$intDate->expects($this->once())->method('getFieldValue')->will($this->returnValue('2013-06-30 23:59:59 +0'));
$messageId = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$messageId->expects($this->once())->method('getFieldValue')->will($this->returnValue('MessageId'));
$xMsgId = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$xMsgId->expects($this->once())->method('getFieldValue')->will($this->returnValue('XMsgId'));
$xThrId = $this->getMock('Zend\\Mail\\Header\\HeaderInterface');
$xThrId->expects($this->once())->method('getFieldValue')->will($this->returnValue('XThrId'));
$toAddress = $this->getMock('Zend\\Mail\\Address\\AddressInterface');
$toAddress->expects($this->once())->method('toString')->will($this->returnValue('toEmail'));
$toAddressList = $this->getMockForAbstractClass('Zend\\Mail\\Header\\AbstractAddressList', array(), '', false, false, true, array('getAddressList'));
$toAddressList->expects($this->once())->method('getAddressList')->will($this->returnValue(array($toAddress)));
$ccAddress = $this->getMock('Zend\\Mail\\Address\\AddressInterface');
$ccAddress->expects($this->once())->method('toString')->will($this->returnValue('ccEmail'));
$ccAddressList = $this->getMockForAbstractClass('Zend\\Mail\\Header\\AbstractAddressList', array(), '', false, false, true, array('getAddressList'));
$ccAddressList->expects($this->once())->method('getAddressList')->will($this->returnValue(array($ccAddress)));
$bccAddress = $this->getMock('Zend\\Mail\\Address\\AddressInterface');
$bccAddress->expects($this->once())->method('toString')->will($this->returnValue('bccEmail'));
$bccAddressList = $this->getMockForAbstractClass('Zend\\Mail\\Header\\AbstractAddressList', array(), '', false, false, true, array('getAddressList'));
$bccAddressList->expects($this->once())->method('getAddressList')->will($this->returnValue(array($bccAddress)));
$this->connector->expects($this->once())->method('getUidValidity')->will($this->returnValue(456));
$msg = $this->getMockBuilder('Oro\\Bundle\\ImapBundle\\Mail\\Storage\\Message')->disableOriginalConstructor()->getMock();
$headers = $this->getMockBuilder('Zend\\Mail\\Headers')->disableOriginalConstructor()->getMock();
$msg->expects($this->once())->method('getHeaders')->will($this->returnValue($headers));
$headers->expects($this->any())->method('get')->will($this->returnValueMap(array(array('UID', $uid), array('Subject', $subject), array('From', $fromEmail), array('Date', $date), array('Received', $received), array('InternalDate', $intDate), array('Importance', false), array('Message-ID', $messageId), array('X-GM-MSG-ID', $xMsgId), array('X-GM-THR-ID', $xThrId), array('X-GM-LABELS', false), array('To', $toAddressList), array('Cc', $ccAddressList), array('Bcc', $bccAddressList))));
$query = $this->getMockBuilder('Oro\\Bundle\\ImapBundle\\Connector\\Search\\SearchQuery')->disableOriginalConstructor()->getMock();
$imap = $this->getMockBuilder('Oro\\Bundle\\ImapBundle\\Mail\\Storage\\Imap')->disableOriginalConstructor()->getMock();
$imap->expects($this->any())->method('getMessage')->will($this->returnValue($msg));
$messageIterator = new ImapMessageIterator($imap, array(1));
$this->connector->expects($this->once())->method('findItems')->with($this->identicalTo($query))->will($this->returnValue($messageIterator));
$this->manager->selectFolder('Test Folder');
$emails = $this->manager->getEmails($query);
$this->assertCount(1, $emails);
$emails->rewind();
$email = $emails->current();
$this->assertEquals(123, $email->getId()->getUid());
$this->assertEquals(456, $email->getId()->getUidValidity());
$this->assertEquals('Subject', $email->getSubject());
$this->assertEquals('fromEmail', $email->getFrom());
$this->assertEquals(new \DateTime('2011-06-30 23:59:59', new \DateTimeZone('UTC')), $email->getSentAt());
$this->assertEquals(new \DateTime('2012-06-30 23:59:59', new \DateTimeZone('UTC')), $email->getReceivedAt());
$this->assertEquals(new \DateTime('2013-06-30 23:59:59', new \DateTimeZone('UTC')), $email->getInternalDate());
$this->assertEquals(0, $email->getImportance());
$this->assertEquals('MessageId', $email->getMessageId());
$this->assertEquals('XMsgId', $email->getXMessageId());
$this->assertEquals('XThrId', $email->getXThreadId());
$toRecipients = $email->getToRecipients();
$this->assertEquals('toEmail', $toRecipients[0]);
$ccRecipients = $email->getCcRecipients();
$this->assertEquals('ccEmail', $ccRecipients[0]);
$bccRecipients = $email->getBccRecipients();
$this->assertEquals('bccEmail', $bccRecipients[0]);
}
