/**
* {@inheritdoc}
*/
public function load(ObjectManager $manager)
{
$criteria = new Criteria();
$criteria->where($criteria->expr()->neq('xThreadId', null));
/** @var QueryBuilder $threadQueryBuilder */
$threadQueryBuilder = $manager->getRepository('OroEmailBundle:Email')->createQueryBuilder('entity');
$threadQueryBuilder->distinct()->select('entity.xThreadId');
$threadQueryBuilder->addCriteria($criteria);
$iterator = new BufferedQueryResultIterator($threadQueryBuilder);
$iterator->setBufferSize(self::BATCH_SIZE);
$itemsCount = 0;
$entities = [];
foreach ($iterator as $threadResult) {
$threadId = $threadResult['xThreadId'];
/** @var QueryBuilder $queryBuilder */
$queryBuilder = $manager->getRepository('OroEmailBundle:Email')->createQueryBuilder('entity');
$criteria = new Criteria();
$criteria->where($criteria->expr()->eq('xThreadId', $threadId));
$criteria->orderBy(['created' => 'ASC']);
$queryBuilder->addCriteria($criteria);
$queryBuilder->setFirstResult(0);
$emails = $queryBuilder->getQuery()->execute();
if (count($emails) > 1) {
$itemsCount++;
$newThread = new EmailThread();
$manager->persist($newThread);
foreach ($emails as $key => $email) {
/** @var Email $email */
if ($key == 0) {
$email->setHead(true);
} else {
$email->setHead(false);
}
$email->setThread($newThread);
$entities[] = $email;
}
} elseif (count($emails) == 1) {
$email = $emails[0];
$email->setHead(true);
$itemsCount++;
$entities[] = $email;
}
if (0 == $itemsCount % self::BATCH_SIZE) {
$this->saveEntities($manager, $entities);
$entities = [];
}
}
if ($itemsCount % self::BATCH_SIZE > 0) {
$this->saveEntities($manager, $entities);
}
}
/**
* Update activity
* @param ObjectManager $manager
*/
public function updateEmailActivityDescription(ObjectManager $manager)
{
/** @var QueryBuilder $activityListBuilder */
$activityListBuilder = $manager->getRepository('OroActivityListBundle:ActivityList')->createQueryBuilder('e');
$iterator = new BufferedQueryResultIterator($activityListBuilder);
$iterator->setBufferSize(self::BATCH_SIZE);
$itemsCount = 0;
$entities = [];
$emailRepository = $manager->getRepository('OroEmailBundle:Email');
$activityProvider = $this->container->get('oro_email.activity_list.provider');
foreach ($iterator as $activity) {
$email = $emailRepository->find($activity->getRelatedActivityId());
if ($email) {
$itemsCount++;
$activity->setDescription($activityProvider->getDescription($email));
$entities[] = $activity;
}
if (0 === $itemsCount % self::BATCH_SIZE) {
$this->saveEntities($manager, $entities);
$entities = [];
}
}
if ($itemsCount % self::BATCH_SIZE > 0) {
$this->saveEntities($manager, $entities);
}
}
public function load(ObjectManager $manager)
{
/** @var EntityManager $manager */
$this->customerRepository = $manager->getRepository('OroCRMMagentoBundle:Customer');
// Calculate lifetime value for all customers
$queryBuilder = $this->customerRepository->createQueryBuilder('customer');
$queryBuilder->select('SUM(
CASE WHEN customerOrder.subtotalAmount IS NOT NULL THEN customerOrder.subtotalAmount ELSE 0 END -
CASE WHEN customerOrder.discountAmount IS NOT NULL THEN ABS(customerOrder.discountAmount) ELSE 0 END
) AS lifetime', 'customer.id as customerId', 'customer.lifetime AS oldLifetime', 'IDENTITY(customer.dataChannel) as dataChannelId')->leftJoin('customer.orders', 'customerOrder', 'WITH', $queryBuilder->expr()->neq($queryBuilder->expr()->lower('customerOrder.status'), ':status'))->groupBy('customer.account, customer.id')->orderBy('customer.account')->setParameter('status', Order::STATUS_CANCELED);
// Get lifetime value only for customers that have canceled orders
$this->addFilterByOrderStatus($queryBuilder, Order::STATUS_CANCELED);
$iterator = new BufferedQueryResultIterator($queryBuilder);
$iterator->setBufferSize(self::BUFFER_SIZE);
$channels = [];
foreach ($iterator as $row) {
if ($row['lifetime'] == $row['oldLifetime']) {
continue;
}
$this->updateCustomerLifetimeValue($row['customerId'], $row['lifetime']);
if (!isset($channels[$row['dataChannelId']])) {
$channels[$row['dataChannelId']] = $row['dataChannelId'];
}
}
foreach ($channels as $channelId) {
/** @var Channel $channel */
$channel = $manager->getReference('OroCRMChannelBundle:Channel', $channelId);
$this->updateLifetimeForAccounts($channel);
}
}
/**
* @param StaticSegment $staticSegment
*
* @return \Iterator|BufferedQueryResultIterator
*/
protected function createSubordinateIterator($staticSegment)
{
$qb = $this->getIteratorQueryBuilder($staticSegment);
$alias = sprintf('%s.id', MarketingListQueryBuilderAdapter::MEMBER_ALIAS);
$qb->select([MarketingListQueryBuilderAdapter::MEMBER_ALIAS . '.id member_id', $staticSegment->getId() . ' static_segment_id', $qb->expr()->literal(StaticSegmentMember::STATE_ADD) . ' state'])->leftJoin(sprintf('%s.segmentMembers', MarketingListQueryBuilderAdapter::MEMBER_ALIAS), 'segmentMembers', Join::WITH, $qb->expr()->eq('segmentMembers.staticSegment', $staticSegment->getId()))->andWhere($qb->expr()->andX($qb->expr()->isNull('segmentMembers.id'), $qb->expr()->isNotNull(sprintf('%s.originId', MarketingListQueryBuilderAdapter::MEMBER_ALIAS)), $qb->expr()->eq(sprintf('%s.subscribersList', MarketingListQueryBuilderAdapter::MEMBER_ALIAS), ':subscribersList')))->setParameter('subscribersList', $staticSegment->getSubscribersList())->orderBy($alias)->groupBy($alias);
$bufferedIterator = new BufferedQueryResultIterator($qb);
$bufferedIterator->setReverse(true);
$bufferedIterator->setBufferSize(self::BUFFER_SIZE);
return $bufferedIterator;
}
/**
* @param StaticSegment $staticSegment
*
* {@inheritdoc}
*/
protected function createSubordinateIterator($staticSegment)
{
$qb = $this->getIteratorQueryBuilder($staticSegment);
$qb->andWhere($qb->expr()->isNull(MarketingListQueryBuilderAdapter::MEMBER_ALIAS . '.id'));
$bufferedIterator = new BufferedQueryResultIterator($qb);
$bufferedIterator->setHydrationMode(AbstractQuery::HYDRATE_ARRAY)->setReverse(true);
$bufferedIterator->setBufferSize(self::BUFFER_SIZE);
return new \CallbackFilterIterator($bufferedIterator, function (&$current) use($staticSegment) {
if (is_array($current)) {
$current['subscribersList_id'] = $staticSegment->getSubscribersList()->getId();
$current['entityClass'] = $staticSegment->getMarketingList()->getEntity();
}
return true;
});
}
/**
* @param StaticSegment $staticSegment
*
* @return \Iterator|BufferedQueryResultIterator
*/
protected function createSubordinateIterator($staticSegment)
{
if (!$this->segmentMemberClassName) {
throw new \InvalidArgumentException('StaticSegmentMember class name must be provided');
}
$qb = $this->getIteratorQueryBuilder($staticSegment);
$identity = MarketingListQueryBuilderAdapter::MEMBER_ALIAS . '.id';
$qb->select($identity)->andWhere($qb->expr()->isNotNull($identity));
$segmentMembersQb = clone $qb;
$segmentMembersQb->resetDQLParts()->select(['staticSegment.id static_segment_id', 'smmb.id member_id', $segmentMembersQb->expr()->literal(StaticSegmentMember::STATE_REMOVE) . ' state'])->from($this->segmentMemberClassName, 'segmentMember')->join('segmentMember.member', 'smmb')->join('segmentMember.staticSegment', 'staticSegment')->andWhere($qb->expr()->andX($qb->expr()->eq('staticSegment.id', $staticSegment->getId()), $segmentMembersQb->expr()->in('smmb.id', $qb->getDQL())));
$bufferedIterator = new BufferedQueryResultIterator($segmentMembersQb);
$bufferedIterator->setReverse(true);
$bufferedIterator->setBufferSize(self::BUFFER_SIZE);
return $bufferedIterator;
}
/**
* Adds activity lists data for existing activity records only if we are in update process
* (check on installed parameter)
*
* @param ObjectManager $manager
* @param string $activityClass Activity class we need to add activity list data
* @param string $ownerField
* @param string $organizationField
*/
public function addActivityListsForActivityClass(ObjectManager $manager, $activityClass, $ownerField = '', $organizationField = '')
{
if ($this->container->hasParameter('installed') && $this->container->getParameter('installed')) {
$provider = $this->container->get('oro_activity_list.provider.chain');
$queryBuilder = $manager->getRepository($activityClass)->createQueryBuilder('entity');
$iterator = new BufferedQueryResultIterator($queryBuilder);
$iterator->setBufferSize(self::BATCH_SIZE);
$itemsCount = 0;
$entities = [];
foreach ($iterator as $entity) {
$entities[] = $entity;
$itemsCount++;
if (0 == $itemsCount % self::BATCH_SIZE) {
$this->saveActivityLists($manager, $provider, $entities, $ownerField, $organizationField);
$entities = [];
}
}
if ($itemsCount % static::BATCH_SIZE > 0) {
$this->saveActivityLists($manager, $provider, $entities, $ownerField, $organizationField);
}
}
}
/**
* {@inheritdoc}
*/
public function getRecipients(EmailRecipientsProviderArgs $args)
{
if (!$args->getRelatedEntity()) {
return [];
}
$relatedEntity = $args->getRelatedEntity();
$relatedEntityClass = ClassUtils::getClass($relatedEntity);
$em = $this->registry->getManagerForClass($relatedEntityClass);
$metadata = $em->getClassMetadata($relatedEntityClass);
$idNames = $metadata->getIdentifierFieldNames();
if (count($idNames) !== 1) {
return [];
}
$propertyAccessor = PropertyAccess::createPropertyAccessor();
$relatedEntityId = $propertyAccessor->getValue($relatedEntity, $idNames[0]);
$recipients = [];
$activities = $this->activityManager->getActivities($relatedEntityClass);
$activityListQb = $this->createActivityListQb($relatedEntityClass, $relatedEntityId);
$activityListDql = $activityListQb->getQuery()->getDQL();
$limit = $args->getLimit();
$activityKeys = array_keys($activities);
foreach ($activityKeys as $class) {
$qb = $this->getRepository($class)->createQueryBuilder('e');
$qb->andWhere($qb->expr()->exists($activityListDql))->setParameter('related_activity_class', $class);
foreach ($activityListQb->getParameters() as $param) {
$qb->setParameter($param->getName(), $param->getValue(), $param->getType());
}
$iterator = new BufferedQueryResultIterator($qb);
$iterator->setBufferSize($limit);
foreach ($iterator as $entity) {
$recipients = array_merge($recipients, EmailRecipientsHelper::filterRecipients($args, $this->relatedEmailsProvider->getRecipients($entity, 2, false, $args->getOrganization())));
$limit -= count($recipients);
if ($limit <= 0) {
break 2;
}
}
}
return $recipients;
}
/**
* Update ActivityList Owner
*
* @param ObjectManager $manager
*/
public function updateActivityListOwner(ObjectManager $manager)
{
if ($this->container->hasParameter('installed') && $this->container->getParameter('installed')) {
/** @var ActivityListChainProvider $activitiesProvider */
$activitiesProvider = $this->container->get('oro_activity_list.provider.chain');
/** @var QueryBuilder $activityListBuilder */
$queryBuilder = $manager->getRepository('OroActivityListBundle:ActivityList')->createQueryBuilder('e');
$iterator = new BufferedQueryResultIterator($queryBuilder);
$iterator->setBufferSize(self::BATCH_SIZE);
$itemsCount = 0;
$entities = [];
foreach ($iterator as $entity) {
$entities[] = $entity;
$itemsCount++;
if (0 === $itemsCount % self::BATCH_SIZE) {
$this->saveActivityListOwner($manager, $activitiesProvider, $entities);
$entities = [];
}
}
if ($itemsCount % static::BATCH_SIZE > 0) {
$this->saveActivityListOwner($manager, $activitiesProvider, $entities);
}
}
}
/**
* @param string $entityName
* @param integer|null $offset
* @param integer|null $limit
* @return int
*/
protected function reindexSingleEntity($entityName, $offset = null, $limit = null)
{
/** @var EntityManager $entityManager */
$entityManager = $this->registry->getManagerForClass($entityName);
$entityManager->getConnection()->getConfiguration()->setSQLLogger(null);
$pk = $entityManager->getClassMetadata($entityName)->getIdentifier();
$orderingsExpr = new OrderBy();
foreach ($pk as $fieldName) {
$orderingsExpr->add('entity.' . $fieldName);
}
$queryBuilder = $entityManager->getRepository($entityName)->createQueryBuilder('entity')->orderBy($orderingsExpr);
if (null !== $offset) {
$queryBuilder->setFirstResult($offset);
}
if (null !== $limit) {
$queryBuilder->setMaxResults($limit);
}
$iterator = new BufferedQueryResultIterator($queryBuilder);
$iterator->setBufferSize(static::BATCH_SIZE);
$itemsCount = 0;
$entities = [];
foreach ($iterator as $entity) {
$entities[] = $entity;
$itemsCount++;
if (0 == $itemsCount % static::BATCH_SIZE) {
$this->save($entities, true);
$entityManager->clear();
$entities = [];
gc_collect_cycles();
}
}
if ($itemsCount % static::BATCH_SIZE > 0) {
$this->save($entities, true);
$entityManager->clear();
}
return $itemsCount;
}
/**
* Clear search all search indexes or for custom entity
*
* @param string $entityName
*/
protected function clearSearchIndexForEntity($entityName)
{
$itemsCount = 0;
$entityManager = $this->registry->getManagerForClass('OroSearchBundle:Item');
$queryBuilder = $this->getIndexRepository()->createQueryBuilder('item')->where('item.entity = :entity')->setParameter('entity', $entityName);
$iterator = new BufferedQueryResultIterator($queryBuilder);
$iterator->setBufferSize(static::BATCH_SIZE);
foreach ($iterator as $entity) {
$itemsCount++;
$entityManager->remove($entity);
if (0 == $itemsCount % static::BATCH_SIZE) {
$entityManager->flush();
$entityManager->clear();
}
}
if ($itemsCount % static::BATCH_SIZE > 0) {
$entityManager->flush();
$entityManager->clear();
}
}
/**
* @param QueryBuilder $customersQueryBuilder
*
* @return BufferedQueryResultIterator
*/
protected function getCustomersIterator(QueryBuilder $customersQueryBuilder)
{
$iterator = new BufferedQueryResultIterator($customersQueryBuilder);
$iterator->setBufferSize(static::READ_BATCH_SIZE);
return $iterator;
}
/**
* @param string $entityName
* @return int
*/
protected function reindexSingleEntity($entityName)
{
/** @var EntityManager $entityManager */
$entityManager = $this->registry->getManagerForClass($entityName);
$queryBuilder = $entityManager->getRepository($entityName)->createQueryBuilder('entity');
$iterator = new BufferedQueryResultIterator($queryBuilder);
$iterator->setBufferSize(static::BATCH_SIZE);
$itemsCount = 0;
$entities = array();
foreach ($iterator as $entity) {
$entities[] = $entity;
$itemsCount++;
if (0 == $itemsCount % static::BATCH_SIZE) {
$this->save($entities, true);
$entityManager->clear();
$entities = array();
}
}
if ($itemsCount % static::BATCH_SIZE > 0) {
$this->save($entities, true);
$entityManager->clear();
}
return $itemsCount;
}
/**
* @param Channel $channel
* @param array $ids
*
* @return BufferedQueryResultIterator|CustomerIdentity[]
*/
protected function getEntitiesByChannel(Channel $channel, array $ids = [])
{
$entityFQCN = $channel->getCustomerIdentity();
$qb = $this->getDoctrineHelper()->getEntityRepository($entityFQCN)->createQueryBuilder('e');
$qb->orderBy(sprintf('e.%s', $this->getDoctrineHelper()->getSingleEntityIdentifierFieldName($entityFQCN)));
$qb->andWhere('e.dataChannel = :dataChannel');
$qb->setParameter('dataChannel', $channel);
if ($ids) {
$qb->andWhere($qb->expr()->in('e.id', ':ids'));
$qb->setParameter('ids', $ids);
}
$iterator = new BufferedQueryResultIterator($qb);
// !!! should be the same as flush batch, will not work otherwise because of detached entities after EM#clear()
$iterator->setBufferSize(self::BATCH_SIZE);
return $iterator;
}
/**
* @param Integration $integration
* @param string $connector
* @param int|null $code
* @return Status[]|\Iterator
*/
public function getConnectorStatuses(Integration $integration, $connector, $code = null)
{
$iterator = new BufferedQueryResultIterator($this->getConnectorStatusesQueryBuilder($integration, $connector, $code));
$iterator->setBufferSize(self::BUFFER_SIZE);
return $iterator;
}
public function testIteratorInReverseDirection()
{
$records = [['a_0' => '1'], ['a_0' => '2'], ['a_0' => '3']];
$actualSqls = [];
$statementCounter = 0;
$statements = [$this->createFetchStatementMock([['sclr_0' => count($records)]]), $this->createFetchStatementMock([$records[0], $records[1]]), $this->createFetchStatementMock([$records[2]])];
$this->getDriverConnectionMock($this->em)->expects($this->any())->method('query')->will($this->returnCallback(function ($sql) use(&$statements, &$statementCounter, &$actualSqls) {
$actualSqls[$statementCounter] = $sql;
$statement = $statements[$statementCounter];
$statementCounter++;
return $statement;
}));
$source = $this->em->createQueryBuilder()->select('o')->from('Stub:Entity', 'o');
$iterator = new BufferedQueryResultIterator($source);
$iterator->setReverse(true);
$iterator->setBufferSize(2);
$this->assertEquals(count($records), $iterator->count());
$count = 0;
foreach ($iterator as $record) {
$this->assertInstanceOf('Oro\\Bundle\\BatchBundle\\Tests\\Unit\\ORM\\Query\\Stub\\Entity', $record);
$this->assertEquals($records[$count]['a_0'], $record->a);
$count++;
}
$this->assertEquals(count($records), $count);
$this->assertCount(3, $actualSqls);
$this->assertEquals('SELECT count(e0_.a) AS sclr_0 FROM Entity e0_', $actualSqls[0]);
$this->assertEquals('SELECT e0_.a AS a_0, e0_.b AS b_1 FROM Entity e0_ LIMIT 2 OFFSET 2', $actualSqls[1]);
$this->assertEquals('SELECT e0_.a AS a_0, e0_.b AS b_1 FROM Entity e0_ LIMIT 2 OFFSET 0', $actualSqls[2]);
}
/**
* @param Channel $channel
*/
protected function updateLifetimeForAccounts(Channel $channel)
{
$lifetimeFields = $this->getLifetimeFieldsMap();
$customerIdentity = $channel->getCustomerIdentity();
if (!isset($lifetimeFields[$customerIdentity])) {
return;
}
$lifetimeFieldName = $lifetimeFields[$customerIdentity];
$accountRepo = $this->em->getRepository('OroCRMAccountBundle:Account');
$accountIterator = new BufferedQueryResultIterator($accountRepo->createQueryBuilder('a')->select('a.id'));
$accountIterator->setBufferSize(self::UPDATE_LIFETIME_READ_BATCH_SIZE);
$accountIds = [];
foreach ($accountIterator as $accountRow) {
$accountIds[] = $accountRow['id'];
if (count($accountIds) === self::UPDATE_LIFETIME_WRITE_BATCH_SIZE) {
$this->updateLifetime($accountIds, $channel, $customerIdentity, $lifetimeFieldName);
$accountIds = [];
}
}
if (count($accountIds) > 0) {
$this->updateLifetime($accountIds, $channel, $customerIdentity, $lifetimeFieldName);
}
}