使用scrapy python使用POST请求获取json响应

Question

我正试图从这个网站获得使用post请求的数据。我在那个网站上找到了帖子的url，但我没有得到相同的回应。

下面是我的代码:

import scrapy
from scrapy.http import request
from scrapy.http.request.form import FormRequest
from scrapy.http import FormRequest
import json
class CodeSpider(scrapy.Spider):
name = 'code'
allowed_domains = ['code.comcom']
start_urls = ['https://technet.rapaport.com/HTTP/JSON/RetailFeed/GetDiamonds.aspx']
def start_requests(self): 
form_data = {"request":{"header":{"raplink_access_key":"e7d7d61946804c579d02dab565371113","domain":"www.sarvadajewels.com"},"body":{"search_type":"white","shapes":["round"],"size_from":0.1,"size_to":100,"color_from":"D","color_to":"M","clarity_from":"IF","clarity_to":"I1","cut_from":"Excellent","cut_to":"Poor","polish_from":"Excellent","polish_to":"Poor","symmetry_from":"Excellent","symmetry_to":"Poor","labs":[],"fancy_colors":[],"price_total_from":0,"price_total_to":7428404930,"page_number":2,"page_size":"60","sort_by":"price","sort_direction":"asc","currency_code":"INR"}}}
request_body = json.dumps(form_data)
yield scrapy.Request('https://technet.rapaport.com/HTTP/JSON/RetailFeed/GetDiamonds.aspx',
method="POST",
body=request_body,
headers={'Content-Type': 'application/json; charset=UTF-8'},callback=self.parse )
def parse(self, response):
with open('test.json', 'w') as file:
file.write(str(response.body)

和我面对这个错误:

{'response': {'header': {'error_code': 1001, 'error_message': 'Invalid format'
}, 'body': {}
}

IS there anyway to get this. 

Answer 1

您使用的标题不正确:

{'Content-Type': 'application/json; charset=UTF-8'}

应该是:

{'Content-Type': 'application/x-www-form-urlencoded'}

完整代码:

import scrapy
from scrapy.http import request
from scrapy.http.request.form import FormRequest
from scrapy.http import FormRequest
import json
class CodeSpider(scrapy.Spider):
name = 'code'
allowed_domains = ['code.comcom']
start_urls = ['https://technet.rapaport.com/HTTP/JSON/RetailFeed/GetDiamonds.aspx']
def start_requests(self): 
form_data = {"request":{"header":{"raplink_access_key":"e7d7d61946804c579d02dab565371113","domain":"www.sarvadajewels.com"},"body":{"search_type":"white","shapes":["round"],"size_from":0.1,"size_to":100,"color_from":"D","color_to":"M","clarity_from":"IF","clarity_to":"I1","cut_from":"Excellent","cut_to":"Poor","polish_from":"Excellent","polish_to":"Poor","symmetry_from":"Excellent","symmetry_to":"Poor","labs":[],"fancy_colors":[],"price_total_from":0,"price_total_to":7428404930,"page_number":2,"page_size":"60","sort_by":"price","sort_direction":"asc","currency_code":"INR"}}}
request_body = json.dumps(form_data)
yield scrapy.Request('https://technet.rapaport.com/HTTP/JSON/RetailFeed/GetDiamonds.aspx',
method="POST",
body=request_body,
headers={'Content-Type': 'application/x-www-form-urlencoded'},callback=self.parse )
def parse(self, response):
yield json.loads(response.text)

同时，scrapy支持用-o标志获取的项来写不同格式的文件。所以你可以用它来代替pythonwrite，试试:

scrapy runspider <spider_name> -o test.json

或

scrapy crawl code -o test.json