/**
* @test
*/
public function deleteDocumentAndCreateNewMessageWhenDocumentIsMoved()
{
$redirect_url = "http://redirect.example.com";
$message = new AMQPMessage();
$message->body = json_encode(['url' => 'https://github.com', 'base_url' => 'https://github.com', 'blacklist' => [], 'metadata' => ['core' => 'core1'], 'whitelist' => []]);
$crawlJob = new CrawlJob($redirect_url, 'https://github.com', [], ['core' => 'core1'], []);
$queue = $this->getMockBuilder('Simgroep\\ConcurrentSpiderBundle\\Queue')->disableOriginalConstructor()->setMethods(['__destruct', 'listen', 'publishJob', 'rejectMessage'])->getMock();
$queue->expects($this->once())->method('rejectMessage')->with($this->equalTo($message));
$queue->expects($this->once())->method('publishJob')->with($this->equalTo($crawlJob));
$indexer = $this->getMockBuilder('Simgroep\\ConcurrentSpiderBundle\\Indexer')->disableOriginalConstructor()->setMethods(['isUrlIndexedAndNotExpired', 'deleteDocument'])->getMock();
$indexer->expects($this->once())->method('isUrlIndexedAndNotExpired')->with($this->equalTo('https://github.com'))->will($this->returnValue(false));
$indexer->expects($this->once())->method('deleteDocument')->with($this->equalTo($message))->will($this->returnValue(false));
$client = $this->getMockBuilder('Guzzle\\Http\\Client')->disableOriginalConstructor()->setMethods(['setUserAgent'])->getMock();
$client->setConfig(new Collection());
$requestHandler = $this->getMockBuilder('VDB\\Spider\\RequestHandler\\GuzzleRequestHandler')->disableOriginalConstructor()->setMethods(['getClient'])->getMock();
$requestHandler->expects($this->exactly(2))->method('getClient')->will($this->returnValue($client));
$response = $this->getMockBuilder(Response::class)->disableOriginalConstructor()->setMethods(['getInfo', 'getStatusCode'])->getMock();
$response->expects($this->once())->method('getStatusCode')->will($this->returnValue("301"));
$response->expects($this->once())->method('getInfo')->with($this->equalTo('redirect_url'))->will($this->returnValue($redirect_url));
$exception = new ClientErrorResponseException(sprintf("Page moved to %s", $redirect_url), 301);
$exception->setResponse($response);
$spider = $this->getMockBuilder('Simgroep\\ConcurrentSpiderBundle\\Spider')->setMethods(['getRequestHandler', 'crawl'])->disableOriginalConstructor()->getMock();
$spider->expects($this->exactly(2))->method('getRequestHandler')->will($this->returnValue($requestHandler));
$spider->expects($this->once())->method("crawl")->willThrowException($exception);
$userAgent = 'I am some agent';
$logger = $this->getMockBuilder('Monolog\\Logger')->disableOriginalConstructor()->setMethods(['info', 'warning', 'emergency'])->getMock();
$command = $this->getMockBuilder('Simgroep\\ConcurrentSpiderBundle\\Command\\CrawlCommand')->setConstructorArgs([$queue, $indexer, $spider, $userAgent, $logger])->setMethods(null)->getMock();
$command->crawlUrl($message);
}
/**
* @expectedException \Guzzle\Http\Exception\ClientErrorResponseException
*/
public function testGetWhereClientExceptionThrownButNot404()
{
$faker = Factory::create();
$key = $faker->unique()->word;
$valueField = $faker->unique()->word;
$mount = $faker->unique()->word;
$apiVersion = $faker->unique()->word;
$exception = new ClientErrorResponseException();
$exception->setResponse(\Mockery::mock('Guzzle\\Http\\Message\\Response')->shouldReceive('getStatusCode')->andReturn(400)->getMock());
$client = \Mockery::mock('Guzzle\\Http\\Client')->shouldReceive('get')->with(sprintf('/%s/%s/%s', $apiVersion, $mount, $key))->andReturn(\Mockery::mock()->shouldReceive('send')->andThrow($exception)->getMock())->getMock();
$source = new VaultSource($client, $mount, $valueField, $apiVersion);
$source->get($key);
}
/**
* @expectedException PayPalRestApiClient\Exception\CallException
* @expectedExceptionMessage Error requesting token: response status 401 Unauthorized, reason: error: invalid_client, error_description: Client secret does not match for this client
*/
public function testWrongResponseGivenShouldThrowPayPalCallException()
{
$clientId = 'example';
$secret = 'example';
$expectedRequest = $this->getMockBuilder('Guzzle\\Http\\Message\\Request')->disableOriginalConstructor()->getMock();
$expectedResponse = $this->getMockBuilder('Guzzle\\Http\\Message\\Response')->disableOriginalConstructor()->setMethods(array('getStatusCode', 'getBody', 'getReasonPhrase'))->getMock();
$expectedResponse->expects($this->any())->method('getReasonPhrase')->will($this->returnValue('Unauthorized'));
$expectedResponse->expects($this->any())->method('getStatusCode')->will($this->returnValue(401));
$expectedResponse->expects($this->any())->method('getBody')->will($this->returnValue('{"error":"invalid_client","error_description":"Client secret does not match for this client"}'));
$this->client = $this->getMockBuilder('Guzzle\\Http\\Client')->disableOriginalConstructor()->setMethods(array('createRequest', 'send'))->getMock();
$this->client->expects($this->once())->method('createRequest')->with('POST', 'https://api.sandbox.paypal.com/v1/oauth2/token', array('Accept' => 'application/json', 'Accept-Language' => 'en_US', 'Content-Type' => 'application/x-www-form-urlencoded'), 'grant_type=client_credentials', array('auth' => array($clientId, $secret), 'debug' => true))->will($this->returnValue($expectedRequest));
$exception = new ClientErrorResponseException();
$exception->setResponse($expectedResponse);
$this->client->expects($this->once())->method('send')->with($expectedRequest)->will($this->throwException($exception));
$repo = new AccessTokenRepository($this->client, 'https://api.sandbox.paypal.com', true);
$token = $repo->getAccessToken($clientId, $secret);
}
/**
* Function that crawls one webpage based on the give url.
*
* @param \Simgroep\ConcurrentSpiderBundle\CrawlJob $crawlJob
*/
public function crawl(CrawlJob $crawlJob)
{
$this->currentCrawlJob = $crawlJob;
$resource = $this->requestHandler->request(new Uri($crawlJob->getUrl()));
if ($resource->getResponse()->getStatusCode() == 301) {
$exception = new ClientErrorResponseException(sprintf("Page moved to %s", $resource->getResponse()->getInfo('redirect_url')), 301);
$exception->setResponse($resource->getResponse());
throw $exception;
}
$uris = [];
$this->eventDispatcher->dispatch(SpiderEvents::SPIDER_CRAWL_PRE_DISCOVER);
$baseUrl = $resource->getUri()->toString();
$crawler = $resource->getCrawler()->filterXPath('//a');
foreach ($crawler as $node) {
try {
if ($node->getAttribute("rel") === "nofollow") {
continue;
}
$href = $node->getAttribute('href');
$uri = new Uri($href, $baseUrl);
$uris[] = $uri;
} catch (UriSyntaxException $e) {
//too bad
}
}
$crawler = $resource->getCrawler()->filterXPath('//loc');
foreach ($crawler as $node) {
try {
$href = $node->nodeValue;
$uri = new Uri($href, $baseUrl);
$uris[] = $uri;
} catch (UriSyntaxException $e) {
//too bad
}
}
$this->eventDispatcher->dispatch(SpiderEvents::SPIDER_CRAWL_POST_DISCOVER, new GenericEvent($this, ['uris' => $uris]));
$this->persistenceHandler->persist($resource, $crawlJob);
}
/**
* @param GuzzleResponse $response
* @return ClientErrorResponseException
*/
private function createGuzzleClientException(GuzzleResponse $response)
{
$e = new ClientErrorResponseException();
$e->setResponse($response);
return $e;
}
