Anurag Misra Anurag Misra - 2 months ago 29
Python Question

Can Server read Request.Meta data sent by scrapy?

Code below is basically a sample of Amazon Spider.

I wanna know can amazon server(or any other server) know what's data are we passing into scrapy Request.meta . If Request.meta is not passed with our request then how do we receive that meta data into our response.meta.

Can someone explain how scrapy request.meta and response.meta work ?

import random
from HTMLParser import HTMLParser

import scrapy
from scrapy.crawler import CrawlerProcess

import os
import sys

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
from amazon.items import AmazonItem
from amazon.user_agents import user_agent_list

class MLStripper(HTMLParser):
def __init__(self):
self.fed = []

def handle_data(self, d):

def get_data(self):
return ''.join(self.fed)

def strip_tags(html):
s = MLStripper()
return s.get_data()

class Amazon(scrapy.Spider):
allowed_domains = ['']
start_urls = ['']
name = 'amazon'

def parse(self, response):
product_detail = response.xpath('//li[@class="s-result-item celwidget "]')
for product in product_detail:
asin = product.xpath('@data-asin').extract_first().encode('ascii', 'ignore')
url = '' + asin
brand = product.xpath('div/div/div/span[2]/text()').extract_first()
if brand != 'Azani':
request = scrapy.Request(url, callback=self.parse_product)
request.meta['asin'] = asin
yield request

next_page = response.xpath('//a[@id="pagnNextLink"]/@href').extract_first()
if next_page:
next_page = '' + next_page
request = scrapy.Request(next_page, callback=self.parse)
yield request

def offer_page(self, response):
item = response.meta['item']
seller = response.xpath('//div[@class="a-row a-spacing-mini olpOffer"]/div/h3/span/a/text()').extract()
price = response.xpath('//div[@class="a-row a-spacing-mini olpOffer"]/div/span/span/text()').extract()
seller_price = zip(seller, price)
item['brand'] = response.xpath('//div[@id="olpProductByline"]/text()').extract_first().strip().replace('by ',
item['price'] = '{}'.format(seller_price)
item['no_of_seller'] = len(seller_price)
yield item

def parse_product(self, response):
def html_to_text(html):
s = MLStripper()
return s.get_data()

asin = response.meta['asin']
item = AmazonItem()
item['asin'] = asin
item['product_name'] = response.xpath('//*[@id="productTitle"]/text()').extract_first().strip()
item['bullet_point'] = html_to_text(
item['description'] = html_to_text(response.xpath('//*[@id="productDescription"]').extract_first()).strip()
child_asins = response.xpath('//*[@class="dropdownAvailable"]/@value').extract()
child_asins = map(lambda x: x.split(',')[-1], child_asins)
child_asins = ','.join(child_asins)
item['child_asin'] = child_asins.encode('utf-8', 'ignore')
offer_page = '' + asin
request = scrapy.Request(offer_page, callback=self.offer_page)
request.meta['item'] = item
yield request

Answer Source


You can see what request sends off to source by inspecting request.body and request.headers attributes.

$ scrapy shell ""
>[1]: request.headers
{b'Accept': b'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 b'Accept-Encoding': b'gzip,deflate',
 b'Accept-Language': b'en',
 b'User-Agent': b'scrapy'}
>[2]: request.body
<[2]: b''
>[3]: request.method
<[3]: 'GET'

meta attribute is only used internally by scrapy to retain some data between requests.
e.g. you make a request to a website with meta={'name':'foo'} then scrapy schedules that request, and once response is ready, it makes a Response object with that meta + some meta info it calculates itself and passes it on to your Request.callback function.