/** * Loads emails from an email server and save them into the database * * @param EmailFolder $folder * @param SearchQuery $searchQuery */ protected function loadEmails(EmailFolder $folder, SearchQuery $searchQuery) { $this->log->notice(sprintf('Query: "%s".', $searchQuery->convertToSearchString())); $folder->setSynchronizedAt(new \DateTime('now', new \DateTimeZone('UTC'))); $emails = $this->manager->getEmails($searchQuery); $needFolderFlush = true; $count = 0; $batch = array(); foreach ($emails as $email) { $count++; $batch[] = $email; if ($count === self::DB_BATCH_SIZE) { $this->saveEmails($batch, $folder); $needFolderFlush = false; $count = 0; $batch = array(); } } if ($count > 0) { $this->saveEmails($batch, $folder); $needFolderFlush = false; } if ($needFolderFlush) { $this->em->flush(); } }
/** * @param Message[] $batch */ protected function handleBatchLoaded($batch) { $this->batch = []; foreach ($batch as $key => $val) { $this->batch[$key] = $this->manager->convertToEmail($val); } }
public function testConvertToEmailWithMultiValueAcceptLanguage() { $msg = $this->getMockBuilder('Oro\\Bundle\\ImapBundle\\Mail\\Storage\\Message')->disableOriginalConstructor()->getMock(); $headers = $this->getMockBuilder('Zend\\Mail\\Headers')->disableOriginalConstructor()->getMock(); $msg->expects($this->once())->method('getHeaders')->will($this->returnValue($headers)); $headers->expects($this->any())->method('get')->will($this->returnValueMap([['UID', $this->getHeader('123')], ['Subject', $this->getHeader('Subject')], ['From', $this->getHeader('fromEmail')], ['Date', $this->getHeader('Fri, 31 Jun 2011 10:59:59 +1100')], ['Received', $this->getHeader('by server to email; Fri, 31 Jun 2011 10:58:58 +1100')], ['InternalDate', $this->getHeader('Fri, 31 Jun 2011 10:57:57 +1100')], ['Message-ID', $this->getHeader('MessageId')], ['Importance', false], ['References', $this->getHeader('References')], ['X-GM-MSG-ID', $this->getHeader('XMsgId')], ['X-GM-THR-ID', $this->getHeader('XThrId1')], ['X-GM-LABELS', false], ['Accept-Language', $this->getMultiValueHeader(['en-US', 'en-US'])]])); $email = $this->manager->convertToEmail($msg); $this->assertNotEmpty($email->getMessageId()); $this->assertEquals('en-US', $email->getAcceptLanguageHeader()); }
/** * {@inheritdoc} */ public function loadEmailBody(EmailFolder $folder, Email $email, EntityManager $em) { /** @var UserEmailOrigin $origin */ $origin = $folder->getOrigin(); $config = new ImapConfig($origin->getImapHost(), $origin->getImapPort(), $origin->getImapEncryption(), $origin->getUser(), $this->encryptor->decryptData($origin->getPassword()), $this->imapEmailGoogleOauth2Manager->getAccessTokenWithCheckingExpiration($origin)); $manager = new ImapEmailManager($this->connectorFactory->createImapConnector($config)); $manager->selectFolder($folder->getFullName()); $repo = $em->getRepository('OroImapBundle:ImapEmail'); $query = $repo->createQueryBuilder('e')->select('e.uid')->innerJoin('e.imapFolder', 'if')->where('e.email = ?1 AND if.folder = ?2')->setParameter(1, $email)->setParameter(2, $folder)->getQuery(); $loadedEmail = $manager->findEmail($query->getSingleScalarResult()); if (null === $loadedEmail) { throw new EmailBodyNotFoundException($email); } $builder = new EmailBodyBuilder(); $builder->setEmailBody($loadedEmail->getBody()->getContent(), $loadedEmail->getBody()->getBodyIsText()); foreach ($loadedEmail->getAttachments() as $attachment) { $builder->addEmailAttachment($attachment->getFileName(), $attachment->getContent(), $attachment->getContentType(), $attachment->getContentTransferEncoding(), $attachment->getContentId()); } return $builder->getEmailBody(); }
/** * {@inheritdoc} */ public function loadEmailBody(Email $email, EntityManager $em) { /** @var ImapEmailOrigin $origin */ $origin = $email->getFolder()->getOrigin(); $config = new ImapConfig($origin->getHost(), $origin->getPort(), $origin->getSsl(), $origin->getUser(), $this->encryptor->decryptData($origin->getPassword())); $manager = new ImapEmailManager($this->connectorFactory->createImapConnector($config)); $manager->selectFolder($email->getFolder()->getFullName()); $repo = $em->getRepository('OroImapBundle:ImapEmail'); $query = $repo->createQueryBuilder('e')->select('e.uid')->where('e.email = ?1')->setParameter(1, $email)->getQuery(); /** @var ImapEmail $imapEmail */ $imapEmail = $query->getSingleResult(); $loadedEmail = $manager->findEmail($imapEmail['uid']); if ($loadedEmail === null) { throw new \RuntimeException(sprintf('Cannot find a body for "%s" email.', $email->getSubject())); } $builder = new EmailBodyBuilder(); $builder->setEmailBody($loadedEmail->getBody()->getContent(), $loadedEmail->getBody()->getBodyIsText()); foreach ($loadedEmail->getAttachments() as $attachment) { $builder->addEmailAttachment($attachment->getFileName(), $attachment->getContent(), $attachment->getContentType(), $attachment->getContentTransferEncoding()); } return $builder->getEmailBody(); }
/** * Saves emails into the database * * @param Email[] $emails * @param ImapEmailFolder $imapFolder */ protected function saveEmails(array $emails, ImapEmailFolder $imapFolder) { $this->emailEntityBuilder->removeEmails(); $folder = $imapFolder->getFolder(); $existingUids = $this->getExistingUids($folder, $emails); $isMultiFolder = $this->manager->hasCapability(Imap::CAPABILITY_MSG_MULTI_FOLDERS); $messageIds = $this->getNewMessageIds($emails, $existingUids); $existingImapEmails = $this->getExistingImapEmails($folder->getOrigin(), $messageIds, $isMultiFolder); $existingEmailUsers = $this->getExistingEmailUsers($folder, $messageIds); /** @var ImapEmail[] $newImapEmails */ $newImapEmails = []; foreach ($emails as $email) { if (!$this->allowSaveEmail($folder, $email, $existingUids)) { continue; } /** @var ImapEmail[] $relatedExistingImapEmails */ $relatedExistingImapEmails = array_filter($existingImapEmails, function (ImapEmail $imapEmail) use($email) { return $imapEmail->getEmail()->getMessageId() === $email->getMessageId(); }); $existingImapEmail = $this->findExistingImapEmail($relatedExistingImapEmails, $folder->getType(), $isMultiFolder); if ($existingImapEmail) { $this->moveEmailToOtherFolder($existingImapEmail, $imapFolder, $email->getId()->getUid()); } else { try { $emailUser = isset($existingEmailUsers[$email->getMessageId()]) ? $existingEmailUsers[$email->getMessageId()] : $this->addEmailUser($email, $folder, $email->hasFlag("\\Seen"), $this->currentUser, $this->currentOrganization); $imapEmail = $this->createImapEmail($email->getId()->getUid(), $emailUser->getEmail(), $imapFolder); $newImapEmails[] = $imapEmail; $this->em->persist($imapEmail); $this->logger->notice(sprintf('The "%s" (UID: %d) email was persisted.', $email->getSubject(), $email->getId()->getUid())); } catch (\Exception $e) { $this->logger->warning(sprintf('Failed to persist "%s" (UID: %d) email. Error: %s', $email->getSubject(), $email->getId()->getUid(), $e->getMessage())); } } $this->removeEmailFromOutdatedFolders($relatedExistingImapEmails); } $this->emailEntityBuilder->getBatch()->persist($this->em); // update references if needed $changes = $this->emailEntityBuilder->getBatch()->getChanges(); foreach ($newImapEmails as $imapEmail) { foreach ($changes as $change) { if ($change['old'] instanceof EmailEntity && $imapEmail->getEmail() === $change['old']) { $imapEmail->setEmail($change['new']); } } } $this->em->flush(); $this->cleanUp(); }
/** * Saves emails into the database * * @param Email[] $emails * @param ImapEmailFolder $imapFolder */ protected function saveEmails(array $emails, ImapEmailFolder $imapFolder) { $this->emailEntityBuilder->removeEmails(); $folder = $imapFolder->getFolder(); $existingUids = $this->getExistingUids($folder, $emails); $isMultiFolder = $this->manager->hasCapability(Imap::CAPABILITY_MSG_MULTI_FOLDERS); $existingImapEmails = $this->getExistingImapEmails($folder->getOrigin(), $this->getNewMessageIds($emails, $existingUids), $isMultiFolder); /** @var ImapEmail[] $newImapEmails */ $newImapEmails = []; foreach ($emails as $email) { if (in_array($email->getId()->getUid(), $existingUids)) { $this->log->notice(sprintf('Skip "%s" (UID: %d) email, because it is already synchronised.', $email->getSubject(), $email->getId()->getUid())); continue; } /** @var ImapEmail[] $relatedExistingImapEmails */ $relatedExistingImapEmails = array_filter($existingImapEmails, function (ImapEmail $imapEmail) use($email) { return $imapEmail->getEmail()->getMessageId() === $email->getMessageId(); }); $existingImapEmail = $this->findExistingImapEmail($relatedExistingImapEmails, $folder->getType(), $isMultiFolder); if ($existingImapEmail) { $this->moveEmailToOtherFolder($existingImapEmail, $imapFolder, $email->getId()->getUid()); } else { $this->log->notice(sprintf('Persisting "%s" email (UID: %d) ...', $email->getSubject(), $email->getId()->getUid())); $imapEmail = $this->createImapEmail($email->getId()->getUid(), $this->addEmail($email, $folder), $imapFolder); $newImapEmails[] = $imapEmail; $this->em->persist($imapEmail); $this->log->notice(sprintf('The "%s" email was persisted.', $email->getSubject())); } $this->removeEmailFromOutdatedFolders($relatedExistingImapEmails); } $this->emailEntityBuilder->getBatch()->persist($this->em); // update references if needed $changes = $this->emailEntityBuilder->getBatch()->getChanges(); foreach ($newImapEmails as $imapEmail) { foreach ($changes as $change) { if ($change['old'] instanceof EmailEntity && $imapEmail->getEmail() === $change['old']) { $imapEmail->setEmail($change['new']); } } } $this->em->flush(); }
/** * @param Message[] $batch * * @throws \Exception */ protected function handleBatchLoaded($batch) { $this->batch = []; $counter = 0; foreach ($batch as $key => $val) { try { $email = $this->manager->convertToEmail($val); } catch (\Exception $e) { if (null !== $this->onConvertError) { call_user_func($this->onConvertError, $e); $email = null; } else { throw $e; } } // do not add invalid messages to $this->batch if ($email !== null) { $this->batch[$this->iterationPos + $counter] = $email; } $counter++; } }
/** * Get email ids and create iterator * * @param EmailOrigin $origin * @param ImapEmailFolder $imapFolder * @param EmailFolder $folder * * @return ImapEmailIterator */ protected function getEmailIterator(EmailOrigin $origin, ImapEmailFolder $imapFolder, EmailFolder $folder) { if ($origin->getMailbox()) { // build search query for emails sync $sqb = $this->manager->getSearchQueryBuilder(); if ($origin->getSynchronizedAt() && $folder->getSynchronizedAt()) { if ($folder->getType() === FolderType::SENT) { $sqb->sent($folder->getSynchronizedAt()); } else { $sqb->received($folder->getSynchronizedAt()); } } $searchQuery = $sqb->get(); $this->logger->info(sprintf('Loading emails from "%s" folder ...', $folder->getFullName())); $this->logger->info(sprintf('Query: "%s".', $searchQuery->convertToSearchString())); $emails = $this->manager->getEmails($searchQuery); } else { $lastUid = $this->em->getRepository('OroImapBundle:ImapEmail')->findLastUidByFolder($imapFolder); $this->logger->info(sprintf('Previous max email UID "%s"', $lastUid)); $emails = $this->manager->getEmailsUidBased(sprintf('%s:*', ++$lastUid)); } return $emails; }
/** * Return the current element * * @return Email */ public function current() { return $this->manager->convertToEmail($this->iterator->current()); }
/** * @SuppressWarnings(PHPMD.ExcessiveMethodLength) */ public function testGetEmails() { $uid = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $uid->expects($this->once())->method('getFieldValue')->will($this->returnValue('123')); $subject = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $subject->expects($this->once())->method('getFieldValue')->will($this->returnValue('Subject')); $fromEmail = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $fromEmail->expects($this->once())->method('getFieldValue')->will($this->returnValue('fromEmail')); $date = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $date->expects($this->once())->method('getFieldValue')->will($this->returnValue('2011-06-30 23:59:59 +0')); $received = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $received->expects($this->once())->method('getFieldValue')->will($this->returnValue('by server to email; 2012-06-30 23:59:59 +0')); $intDate = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $intDate->expects($this->once())->method('getFieldValue')->will($this->returnValue('2013-06-30 23:59:59 +0')); $messageId = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $messageId->expects($this->once())->method('getFieldValue')->will($this->returnValue('MessageId')); $xMsgId = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $xMsgId->expects($this->once())->method('getFieldValue')->will($this->returnValue('XMsgId')); $xThrId = $this->getMock('Zend\\Mail\\Header\\HeaderInterface'); $xThrId->expects($this->once())->method('getFieldValue')->will($this->returnValue('XThrId')); $toAddress = $this->getMock('Zend\\Mail\\Address\\AddressInterface'); $toAddress->expects($this->once())->method('toString')->will($this->returnValue('toEmail')); $toAddressList = $this->getMockForAbstractClass('Zend\\Mail\\Header\\AbstractAddressList', array(), '', false, false, true, array('getAddressList')); $toAddressList->expects($this->once())->method('getAddressList')->will($this->returnValue(array($toAddress))); $ccAddress = $this->getMock('Zend\\Mail\\Address\\AddressInterface'); $ccAddress->expects($this->once())->method('toString')->will($this->returnValue('ccEmail')); $ccAddressList = $this->getMockForAbstractClass('Zend\\Mail\\Header\\AbstractAddressList', array(), '', false, false, true, array('getAddressList')); $ccAddressList->expects($this->once())->method('getAddressList')->will($this->returnValue(array($ccAddress))); $bccAddress = $this->getMock('Zend\\Mail\\Address\\AddressInterface'); $bccAddress->expects($this->once())->method('toString')->will($this->returnValue('bccEmail')); $bccAddressList = $this->getMockForAbstractClass('Zend\\Mail\\Header\\AbstractAddressList', array(), '', false, false, true, array('getAddressList')); $bccAddressList->expects($this->once())->method('getAddressList')->will($this->returnValue(array($bccAddress))); $this->connector->expects($this->once())->method('getUidValidity')->will($this->returnValue(456)); $msg = $this->getMockBuilder('Oro\\Bundle\\ImapBundle\\Mail\\Storage\\Message')->disableOriginalConstructor()->getMock(); $headers = $this->getMockBuilder('Zend\\Mail\\Headers')->disableOriginalConstructor()->getMock(); $msg->expects($this->once())->method('getHeaders')->will($this->returnValue($headers)); $headers->expects($this->any())->method('get')->will($this->returnValueMap(array(array('UID', $uid), array('Subject', $subject), array('From', $fromEmail), array('Date', $date), array('Received', $received), array('InternalDate', $intDate), array('Importance', false), array('Message-ID', $messageId), array('X-GM-MSG-ID', $xMsgId), array('X-GM-THR-ID', $xThrId), array('X-GM-LABELS', false), array('To', $toAddressList), array('Cc', $ccAddressList), array('Bcc', $bccAddressList)))); $query = $this->getMockBuilder('Oro\\Bundle\\ImapBundle\\Connector\\Search\\SearchQuery')->disableOriginalConstructor()->getMock(); $imap = $this->getMockBuilder('Oro\\Bundle\\ImapBundle\\Mail\\Storage\\Imap')->disableOriginalConstructor()->getMock(); $imap->expects($this->any())->method('getMessage')->will($this->returnValue($msg)); $messageIterator = new ImapMessageIterator($imap, array(1)); $this->connector->expects($this->once())->method('findItems')->with($this->identicalTo($query))->will($this->returnValue($messageIterator)); $this->manager->selectFolder('Test Folder'); $emails = $this->manager->getEmails($query); $this->assertCount(1, $emails); $emails->rewind(); $email = $emails->current(); $this->assertEquals(123, $email->getId()->getUid()); $this->assertEquals(456, $email->getId()->getUidValidity()); $this->assertEquals('Subject', $email->getSubject()); $this->assertEquals('fromEmail', $email->getFrom()); $this->assertEquals(new \DateTime('2011-06-30 23:59:59', new \DateTimeZone('UTC')), $email->getSentAt()); $this->assertEquals(new \DateTime('2012-06-30 23:59:59', new \DateTimeZone('UTC')), $email->getReceivedAt()); $this->assertEquals(new \DateTime('2013-06-30 23:59:59', new \DateTimeZone('UTC')), $email->getInternalDate()); $this->assertEquals(0, $email->getImportance()); $this->assertEquals('MessageId', $email->getMessageId()); $this->assertEquals('XMsgId', $email->getXMessageId()); $this->assertEquals('XThrId', $email->getXThreadId()); $toRecipients = $email->getToRecipients(); $this->assertEquals('toEmail', $toRecipients[0]); $ccRecipients = $email->getCcRecipients(); $this->assertEquals('ccEmail', $ccRecipients[0]); $bccRecipients = $email->getBccRecipients(); $this->assertEquals('bccEmail', $bccRecipients[0]); }