/** * {@inheritdoc} */ public function load(ObjectManager $manager) { $criteria = new Criteria(); $criteria->where($criteria->expr()->neq('xThreadId', null)); /** @var QueryBuilder $threadQueryBuilder */ $threadQueryBuilder = $manager->getRepository('OroEmailBundle:Email')->createQueryBuilder('entity'); $threadQueryBuilder->distinct()->select('entity.xThreadId'); $threadQueryBuilder->addCriteria($criteria); $iterator = new BufferedQueryResultIterator($threadQueryBuilder); $iterator->setBufferSize(self::BATCH_SIZE); $itemsCount = 0; $entities = []; foreach ($iterator as $threadResult) { $threadId = $threadResult['xThreadId']; /** @var QueryBuilder $queryBuilder */ $queryBuilder = $manager->getRepository('OroEmailBundle:Email')->createQueryBuilder('entity'); $criteria = new Criteria(); $criteria->where($criteria->expr()->eq('xThreadId', $threadId)); $criteria->orderBy(['created' => 'ASC']); $queryBuilder->addCriteria($criteria); $queryBuilder->setFirstResult(0); $emails = $queryBuilder->getQuery()->execute(); if (count($emails) > 1) { $itemsCount++; $newThread = new EmailThread(); $manager->persist($newThread); foreach ($emails as $key => $email) { /** @var Email $email */ if ($key == 0) { $email->setHead(true); } else { $email->setHead(false); } $email->setThread($newThread); $entities[] = $email; } } elseif (count($emails) == 1) { $email = $emails[0]; $email->setHead(true); $itemsCount++; $entities[] = $email; } if (0 == $itemsCount % self::BATCH_SIZE) { $this->saveEntities($manager, $entities); $entities = []; } } if ($itemsCount % self::BATCH_SIZE > 0) { $this->saveEntities($manager, $entities); } }
/** * @param StaticSegment $staticSegment * * {@inheritdoc} */ protected function createSubordinateIterator($staticSegment) { $this->assertRequiredDependencies(); if (!$staticSegment->getExtendedMergeVars()) { return new \EmptyIterator(); } $qb = $this->getIteratorQueryBuilder($staticSegment); $marketingList = $staticSegment->getMarketingList(); $memberIdentifier = MarketingListQueryBuilderAdapter::MEMBER_ALIAS . '.id'; $fieldExpr = $this->fieldHelper->getFieldExpr($marketingList->getEntity(), $qb, $this->doctrineHelper->getSingleEntityIdentifierFieldName($marketingList->getEntity())); $qb->addSelect($fieldExpr . ' AS entity_id'); $qb->addSelect($memberIdentifier . ' AS member_id'); $qb->andWhere($qb->expr()->andX($qb->expr()->isNotNull($memberIdentifier))); $bufferedIterator = new BufferedQueryResultIterator($qb); $bufferedIterator->setHydrationMode(AbstractQuery::HYDRATE_ARRAY)->setReverse(true); $uniqueMembers =& $this->uniqueMembers; return new \CallbackFilterIterator($bufferedIterator, function (&$current) use($staticSegment, &$uniqueMembers) { if (is_array($current)) { if (!empty($current['member_id']) && in_array($current['member_id'], $uniqueMembers, true)) { return false; } $current['subscribersList_id'] = $staticSegment->getSubscribersList()->getId(); $current['static_segment_id'] = $staticSegment->getId(); $uniqueMembers[] = $current['member_id']; unset($current['id']); } return true; }); }
public function load(ObjectManager $manager) { /** @var EntityManager $manager */ $this->customerRepository = $manager->getRepository('OroCRMMagentoBundle:Customer'); // Calculate lifetime value for all customers $queryBuilder = $this->customerRepository->createQueryBuilder('customer'); $queryBuilder->select('SUM( CASE WHEN customerOrder.subtotalAmount IS NOT NULL THEN customerOrder.subtotalAmount ELSE 0 END - CASE WHEN customerOrder.discountAmount IS NOT NULL THEN ABS(customerOrder.discountAmount) ELSE 0 END ) AS lifetime', 'customer.id as customerId', 'customer.lifetime AS oldLifetime', 'IDENTITY(customer.dataChannel) as dataChannelId')->leftJoin('customer.orders', 'customerOrder', 'WITH', $queryBuilder->expr()->neq($queryBuilder->expr()->lower('customerOrder.status'), ':status'))->groupBy('customer.account, customer.id')->orderBy('customer.account')->setParameter('status', Order::STATUS_CANCELED); // Get lifetime value only for customers that have canceled orders $this->addFilterByOrderStatus($queryBuilder, Order::STATUS_CANCELED); $iterator = new BufferedQueryResultIterator($queryBuilder); $iterator->setBufferSize(self::BUFFER_SIZE); $channels = []; foreach ($iterator as $row) { if ($row['lifetime'] == $row['oldLifetime']) { continue; } $this->updateCustomerLifetimeValue($row['customerId'], $row['lifetime']); if (!isset($channels[$row['dataChannelId']])) { $channels[$row['dataChannelId']] = $row['dataChannelId']; } } foreach ($channels as $channelId) { /** @var Channel $channel */ $channel = $manager->getReference('OroCRMChannelBundle:Channel', $channelId); $this->updateLifetimeForAccounts($channel); } }
/** * Update activity * @param ObjectManager $manager */ public function updateEmailActivityDescription(ObjectManager $manager) { /** @var QueryBuilder $activityListBuilder */ $activityListBuilder = $manager->getRepository('OroActivityListBundle:ActivityList')->createQueryBuilder('e'); $iterator = new BufferedQueryResultIterator($activityListBuilder); $iterator->setBufferSize(self::BATCH_SIZE); $itemsCount = 0; $entities = []; $emailRepository = $manager->getRepository('OroEmailBundle:Email'); $activityProvider = $this->container->get('oro_email.activity_list.provider'); foreach ($iterator as $activity) { $email = $emailRepository->find($activity->getRelatedActivityId()); if ($email) { $itemsCount++; $activity->setDescription($activityProvider->getDescription($email)); $entities[] = $activity; } if (0 === $itemsCount % self::BATCH_SIZE) { $this->saveEntities($manager, $entities); $entities = []; } } if ($itemsCount % self::BATCH_SIZE > 0) { $this->saveEntities($manager, $entities); } }
/** * @param StaticSegment $staticSegment * * @return \Iterator|BufferedQueryResultIterator */ protected function createSubordinateIterator($staticSegment) { $qb = $this->getIteratorQueryBuilder($staticSegment); $alias = sprintf('%s.id', MarketingListQueryBuilderAdapter::MEMBER_ALIAS); $qb->select([MarketingListQueryBuilderAdapter::MEMBER_ALIAS . '.id member_id', $staticSegment->getId() . ' static_segment_id', $qb->expr()->literal(StaticSegmentMember::STATE_ADD) . ' state'])->leftJoin(sprintf('%s.segmentMembers', MarketingListQueryBuilderAdapter::MEMBER_ALIAS), 'segmentMembers', Join::WITH, $qb->expr()->eq('segmentMembers.staticSegment', $staticSegment->getId()))->andWhere($qb->expr()->andX($qb->expr()->isNull('segmentMembers.id'), $qb->expr()->isNotNull(sprintf('%s.originId', MarketingListQueryBuilderAdapter::MEMBER_ALIAS)), $qb->expr()->eq(sprintf('%s.subscribersList', MarketingListQueryBuilderAdapter::MEMBER_ALIAS), ':subscribersList')))->setParameter('subscribersList', $staticSegment->getSubscribersList())->orderBy($alias)->groupBy($alias); $bufferedIterator = new BufferedQueryResultIterator($qb); $bufferedIterator->setReverse(true); $bufferedIterator->setBufferSize(self::BUFFER_SIZE); return $bufferedIterator; }
/** * {@inheritdoc} */ protected function createSubordinateIterator($subscribersList) { parent::assertSubscribersList($subscribersList); if (!$this->memberClassName) { throw new \InvalidArgumentException('Member id must be provided'); } $qb = $this->doctrineHelper->getEntityManager($this->memberClassName)->getRepository($this->memberClassName)->createQueryBuilder('mmb'); $qb->select('mmb')->join('mmb.subscribersList', 'subscribersList')->where($qb->expr()->andX($qb->expr()->eq('mmb.status', ':status'), $qb->expr()->eq('subscribersList.originId', ':originId')))->setParameters(['status' => Member::STATUS_EXPORT, 'originId' => $subscribersList->getOriginId()])->addOrderBy('subscribersList.id'); $bufferedIterator = new BufferedQueryResultIterator($qb); $bufferedIterator->setReverse(true); return $bufferedIterator; }
/** * @param StaticSegment $staticSegment * * @return \Iterator|BufferedQueryResultIterator */ protected function createSubordinateIterator($staticSegment) { if (!$this->segmentMemberClassName) { throw new \InvalidArgumentException('StaticSegmentMember class name must be provided'); } $qb = $this->getIteratorQueryBuilder($staticSegment)->select(MarketingListQueryBuilderAdapter::MEMBER_ALIAS . '.id'); $segmentMembersQb = clone $qb; $segmentMembersQb->resetDQLParts()->select(['staticSegment.id static_segment_id', 'smmb.id member_id', $segmentMembersQb->expr()->literal(StaticSegmentMember::STATE_UNSUBSCRIBE_DELETE) . ' state'])->from($this->segmentMemberClassName, 'segmentMember')->join('segmentMember.member', 'smmb')->join('segmentMember.staticSegment', 'staticSegment')->where($qb->expr()->andX($qb->expr()->eq('staticSegment.id', $staticSegment->getId()), $segmentMembersQb->expr()->notIn('smmb.id', $qb->getDQL()))); $bufferedIterator = new BufferedQueryResultIterator($segmentMembersQb); $bufferedIterator->setReverse(true); return $bufferedIterator; }
/** * {@inheritdoc} */ protected function createSubordinateIterator($staticSegment) { $vars = $this->provider->provideExtendedMergeVars($staticSegment->getMarketingList()); $varNames = array_map(function ($each) { return $each['name']; }, $vars); $qb = $this->doctrineHelper->getEntityManager($this->extendedMergeVarClassName)->getRepository($this->extendedMergeVarClassName)->createQueryBuilder('extendedMergeVar'); $qb->select(['extendedMergeVar.id', $staticSegment->getId() . ' static_segment_id', 'extendedMergeVar.name', $qb->expr()->literal(ExtendedMergeVar::STATE_REMOVE) . ' state']); $qb->where($qb->expr()->andX($qb->expr()->eq('extendedMergeVar.staticSegment', ':staticSegment'), $qb->expr()->notIn('extendedMergeVar.name', ':vars'), $qb->expr()->neq('extendedMergeVar.state', ':state')))->setParameter('staticSegment', $staticSegment)->setParameter('vars', $varNames)->setParameter('state', ExtendedMergeVar::STATE_DROPPED); $bufferedIterator = new BufferedQueryResultIterator($qb); $bufferedIterator->setHydrationMode(AbstractQuery::HYDRATE_ARRAY)->setReverse(true); return $bufferedIterator; }
/** * @param StaticSegment $staticSegment * * {@inheritdoc} */ protected function createSubordinateIterator($staticSegment) { $qb = $this->getIteratorQueryBuilder($staticSegment); $qb->andWhere($qb->expr()->isNull(MarketingListQueryBuilderAdapter::MEMBER_ALIAS . '.id')); $bufferedIterator = new BufferedQueryResultIterator($qb); $bufferedIterator->setHydrationMode(AbstractQuery::HYDRATE_ARRAY)->setReverse(true); $bufferedIterator->setBufferSize(self::BUFFER_SIZE); return new \CallbackFilterIterator($bufferedIterator, function (&$current) use($staticSegment) { if (is_array($current)) { $current['subscribersList_id'] = $staticSegment->getSubscribersList()->getId(); $current['entityClass'] = $staticSegment->getMarketingList()->getEntity(); } return true; }); }
/** * {@inheritdoc} */ public function getRecipients(EmailRecipientsProviderArgs $args) { if (!$args->getRelatedEntity()) { return []; } $relatedEntity = $args->getRelatedEntity(); $relatedEntityClass = ClassUtils::getClass($relatedEntity); $em = $this->registry->getManagerForClass($relatedEntityClass); $metadata = $em->getClassMetadata($relatedEntityClass); $idNames = $metadata->getIdentifierFieldNames(); if (count($idNames) !== 1) { return []; } $propertyAccessor = PropertyAccess::createPropertyAccessor(); $relatedEntityId = $propertyAccessor->getValue($relatedEntity, $idNames[0]); $recipients = []; $activities = $this->activityManager->getActivities($relatedEntityClass); $activityListQb = $this->createActivityListQb($relatedEntityClass, $relatedEntityId); $activityListDql = $activityListQb->getQuery()->getDQL(); $limit = $args->getLimit(); $activityKeys = array_keys($activities); foreach ($activityKeys as $class) { $qb = $this->getRepository($class)->createQueryBuilder('e'); $qb->andWhere($qb->expr()->exists($activityListDql))->setParameter('related_activity_class', $class); foreach ($activityListQb->getParameters() as $param) { $qb->setParameter($param->getName(), $param->getValue(), $param->getType()); } $iterator = new BufferedQueryResultIterator($qb); $iterator->setBufferSize($limit); foreach ($iterator as $entity) { $recipients = array_merge($recipients, EmailRecipientsHelper::filterRecipients($args, $this->relatedEmailsProvider->getRecipients($entity, 2, false, $args->getOrganization()))); $limit -= count($recipients); if ($limit <= 0) { break 2; } } } return $recipients; }
/** * Adds activity lists data for existing activity records only if we are in update process * (check on installed parameter) * * @param ObjectManager $manager * @param string $activityClass Activity class we need to add activity list data * @param string $ownerField * @param string $organizationField */ public function addActivityListsForActivityClass(ObjectManager $manager, $activityClass, $ownerField = '', $organizationField = '') { if ($this->container->hasParameter('installed') && $this->container->getParameter('installed')) { $provider = $this->container->get('oro_activity_list.provider.chain'); $queryBuilder = $manager->getRepository($activityClass)->createQueryBuilder('entity'); $iterator = new BufferedQueryResultIterator($queryBuilder); $iterator->setBufferSize(self::BATCH_SIZE); $itemsCount = 0; $entities = []; foreach ($iterator as $entity) { $entities[] = $entity; $itemsCount++; if (0 == $itemsCount % self::BATCH_SIZE) { $this->saveActivityLists($manager, $provider, $entities, $ownerField, $organizationField); $entities = []; } } if ($itemsCount % static::BATCH_SIZE > 0) { $this->saveActivityLists($manager, $provider, $entities, $ownerField, $organizationField); } } }
/** * Update ActivityList Owner * * @param ObjectManager $manager */ public function updateActivityListOwner(ObjectManager $manager) { if ($this->container->hasParameter('installed') && $this->container->getParameter('installed')) { /** @var ActivityListChainProvider $activitiesProvider */ $activitiesProvider = $this->container->get('oro_activity_list.provider.chain'); /** @var QueryBuilder $activityListBuilder */ $queryBuilder = $manager->getRepository('OroActivityListBundle:ActivityList')->createQueryBuilder('e'); $iterator = new BufferedQueryResultIterator($queryBuilder); $iterator->setBufferSize(self::BATCH_SIZE); $itemsCount = 0; $entities = []; foreach ($iterator as $entity) { $entities[] = $entity; $itemsCount++; if (0 === $itemsCount % self::BATCH_SIZE) { $this->saveActivityListOwner($manager, $activitiesProvider, $entities); $entities = []; } } if ($itemsCount % static::BATCH_SIZE > 0) { $this->saveActivityListOwner($manager, $activitiesProvider, $entities); } } }
/** * @param string $entityName * @param integer|null $offset * @param integer|null $limit * @return int */ protected function reindexSingleEntity($entityName, $offset = null, $limit = null) { /** @var EntityManager $entityManager */ $entityManager = $this->registry->getManagerForClass($entityName); $entityManager->getConnection()->getConfiguration()->setSQLLogger(null); $pk = $entityManager->getClassMetadata($entityName)->getIdentifier(); $orderingsExpr = new OrderBy(); foreach ($pk as $fieldName) { $orderingsExpr->add('entity.' . $fieldName); } $queryBuilder = $entityManager->getRepository($entityName)->createQueryBuilder('entity')->orderBy($orderingsExpr); if (null !== $offset) { $queryBuilder->setFirstResult($offset); } if (null !== $limit) { $queryBuilder->setMaxResults($limit); } $iterator = new BufferedQueryResultIterator($queryBuilder); $iterator->setBufferSize(static::BATCH_SIZE); $itemsCount = 0; $entities = []; foreach ($iterator as $entity) { $entities[] = $entity; $itemsCount++; if (0 == $itemsCount % static::BATCH_SIZE) { $this->save($entities, true); $entityManager->clear(); $entities = []; gc_collect_cycles(); } } if ($itemsCount % static::BATCH_SIZE > 0) { $this->save($entities, true); $entityManager->clear(); } return $itemsCount; }
/** * @param Channel $channel * @param array $ids * * @return BufferedQueryResultIterator|CustomerIdentity[] */ protected function getEntitiesByChannel(Channel $channel, array $ids = []) { $entityFQCN = $channel->getCustomerIdentity(); $qb = $this->getDoctrineHelper()->getEntityRepository($entityFQCN)->createQueryBuilder('e'); $qb->orderBy(sprintf('e.%s', $this->getDoctrineHelper()->getSingleEntityIdentifierFieldName($entityFQCN))); $qb->andWhere('e.dataChannel = :dataChannel'); $qb->setParameter('dataChannel', $channel); if ($ids) { $qb->andWhere($qb->expr()->in('e.id', ':ids')); $qb->setParameter('ids', $ids); } $iterator = new BufferedQueryResultIterator($qb); // !!! should be the same as flush batch, will not work otherwise because of detached entities after EM#clear() $iterator->setBufferSize(self::BATCH_SIZE); return $iterator; }
/** * Clear search all search indexes or for custom entity * * @param string $entityName */ protected function clearSearchIndexForEntity($entityName) { $itemsCount = 0; $entityManager = $this->registry->getManagerForClass('OroSearchBundle:Item'); $queryBuilder = $this->getIndexRepository()->createQueryBuilder('item')->where('item.entity = :entity')->setParameter('entity', $entityName); $iterator = new BufferedQueryResultIterator($queryBuilder); $iterator->setBufferSize(static::BATCH_SIZE); foreach ($iterator as $entity) { $itemsCount++; $entityManager->remove($entity); if (0 == $itemsCount % static::BATCH_SIZE) { $entityManager->flush(); $entityManager->clear(); } } if ($itemsCount % static::BATCH_SIZE > 0) { $entityManager->flush(); $entityManager->clear(); } }
/** * @param QueryBuilder $customersQueryBuilder * * @return BufferedQueryResultIterator */ protected function getCustomersIterator(QueryBuilder $customersQueryBuilder) { $iterator = new BufferedQueryResultIterator($customersQueryBuilder); $iterator->setBufferSize(static::READ_BATCH_SIZE); return $iterator; }
/** * @param Channel $channel */ protected function updateLifetimeForAccounts(Channel $channel) { $lifetimeFields = $this->getLifetimeFieldsMap(); $customerIdentity = $channel->getCustomerIdentity(); if (!isset($lifetimeFields[$customerIdentity])) { return; } $lifetimeFieldName = $lifetimeFields[$customerIdentity]; $accountRepo = $this->em->getRepository('OroCRMAccountBundle:Account'); $accountIterator = new BufferedQueryResultIterator($accountRepo->createQueryBuilder('a')->select('a.id')); $accountIterator->setBufferSize(self::UPDATE_LIFETIME_READ_BATCH_SIZE); $accountIds = []; foreach ($accountIterator as $accountRow) { $accountIds[] = $accountRow['id']; if (count($accountIds) === self::UPDATE_LIFETIME_WRITE_BATCH_SIZE) { $this->updateLifetime($accountIds, $channel, $customerIdentity, $lifetimeFieldName); $accountIds = []; } } if (count($accountIds) > 0) { $this->updateLifetime($accountIds, $channel, $customerIdentity, $lifetimeFieldName); } }
/** * @param string $entityName * @return int */ protected function reindexSingleEntity($entityName) { /** @var EntityManager $entityManager */ $entityManager = $this->registry->getManagerForClass($entityName); $queryBuilder = $entityManager->getRepository($entityName)->createQueryBuilder('entity'); $iterator = new BufferedQueryResultIterator($queryBuilder); $iterator->setBufferSize(static::BATCH_SIZE); $itemsCount = 0; $entities = array(); foreach ($iterator as $entity) { $entities[] = $entity; $itemsCount++; if (0 == $itemsCount % static::BATCH_SIZE) { $this->save($entities, true); $entityManager->clear(); $entities = array(); } } if ($itemsCount % static::BATCH_SIZE > 0) { $this->save($entities, true); $entityManager->clear(); } return $itemsCount; }
/** * @param Integration $integration * @param string $connector * @param int|null $code * @return Status[]|\Iterator */ public function getConnectorStatuses(Integration $integration, $connector, $code = null) { $iterator = new BufferedQueryResultIterator($this->getConnectorStatusesQueryBuilder($integration, $connector, $code)); $iterator->setBufferSize(self::BUFFER_SIZE); return $iterator; }
/** * {@inheritDoc} */ public function next() { parent::next(); $current = parent::current(); $this->current = $current === null ? null : new ResultRecord($current); }
public function testIteratorInReverseDirection() { $records = [['a_0' => '1'], ['a_0' => '2'], ['a_0' => '3']]; $actualSqls = []; $statementCounter = 0; $statements = [$this->createFetchStatementMock([['sclr_0' => count($records)]]), $this->createFetchStatementMock([$records[0], $records[1]]), $this->createFetchStatementMock([$records[2]])]; $this->getDriverConnectionMock($this->em)->expects($this->any())->method('query')->will($this->returnCallback(function ($sql) use(&$statements, &$statementCounter, &$actualSqls) { $actualSqls[$statementCounter] = $sql; $statement = $statements[$statementCounter]; $statementCounter++; return $statement; })); $source = $this->em->createQueryBuilder()->select('o')->from('Stub:Entity', 'o'); $iterator = new BufferedQueryResultIterator($source); $iterator->setReverse(true); $iterator->setBufferSize(2); $this->assertEquals(count($records), $iterator->count()); $count = 0; foreach ($iterator as $record) { $this->assertInstanceOf('Oro\\Bundle\\BatchBundle\\Tests\\Unit\\ORM\\Query\\Stub\\Entity', $record); $this->assertEquals($records[$count]['a_0'], $record->a); $count++; } $this->assertEquals(count($records), $count); $this->assertCount(3, $actualSqls); $this->assertEquals('SELECT count(e0_.a) AS sclr_0 FROM Entity e0_', $actualSqls[0]); $this->assertEquals('SELECT e0_.a AS a_0, e0_.b AS b_1 FROM Entity e0_ LIMIT 2 OFFSET 2', $actualSqls[1]); $this->assertEquals('SELECT e0_.a AS a_0, e0_.b AS b_1 FROM Entity e0_ LIMIT 2 OFFSET 0', $actualSqls[2]); }
public function testIteratorWithArrayHydrationMode() { $records = [['a0' => '1'], ['a0' => '2'], ['a0' => '3']]; $actualSqls = []; $statementCounter = 0; $statements = [$this->createFetchStatementMock([['sclr0' => count($records)]]), $this->createFetchStatementMock([$records[0], $records[1], $records[2]])]; $this->getDriverConnectionMock($this->em)->expects($this->any())->method('query')->will($this->returnCallback(function ($sql) use(&$statements, &$statementCounter, &$actualSqls) { $actualSqls[$statementCounter] = $sql; $statement = $statements[$statementCounter]; $statementCounter++; return $statement; })); $source = $this->em->createQueryBuilder()->select('o')->from('Stub:Entity', 'o'); $iterator = new BufferedQueryResultIterator($source); $iterator->setHydrationMode(Query::HYDRATE_ARRAY); $this->assertEquals(count($records), $iterator->count()); $count = 0; foreach ($iterator as $record) { $this->assertEquals($records[$count]['a0'], $record['a']); $count++; } $this->assertEquals(count($records), $count); $this->assertEquals('SELECT count(e0_.a) AS sclr0 FROM Entity e0_', $actualSqls[0]); $this->assertEquals('SELECT e0_.a AS a0, e0_.b AS b1 FROM Entity e0_ LIMIT ' . BufferedQueryResultIterator::DEFAULT_BUFFER_SIZE . ' OFFSET 0', $actualSqls[1]); }