属性错误："set"对象没有属性"getall"

Question

我有一个棘手的项目，我试图用有限的经验来完成。我已经完成了nextPage和innerPage操作。我的代码分别搜索广告，并从中逐个抓取数据。之后，它将移动到下一页。我用一个或两个变量检查了所有步骤。

import scrapy
from urllib.parse import urljoin
from scrapy import signals
from pydispatch import dispatcher
import json
import time
class AtasehirSpider(scrapy.Spider):
name = 'atasehir'
allowed_domains = ['www.sahibinden.com','sahibinden.com']
start_urls = ['https://www.sahibinden.com/satilik-daire/istanbul-atasehir-ornek-ornek-mh.?address_region=2&sorting=price_asc&price_min=805000&price_max=900000']
def parse(self, response):
for ad in response.xpath("//td[@class='searchResultsLargeThumbnail']/a/@href").getall():
time.sleep(10)
ads = response.urljoin(ad)
yield response.follow(url=ads, callback=self.parseInnerPage)


next_page_url = response.xpath("//ul[@class='pageNaviButtons']/li/a[@title='Sonraki']/@href").extract_first()
nextPage = response.urljoin(next_page_url)
if nextPage is not None:
time.sleep(10)
yield scrapy.Request(nextPage)

def parseInnerPage(self, response):
ilan_no = response.xpath("//ul[@class='classifiedInfoList']/li[1]/span/text()").get()
ilan_tarihi = response.xpath("//ul[@class='classifiedInfoList']/li[2]/span/text()").get()
emlak_tipi = response.xpath("//ul[@class='classifiedInfoList']/li[3]/span/text()").get()
metrekare_brut = response.xpath("//ul[@class='classifiedInfoList']/li[4]/span/text()").get()
metrekare_net = response.xpath("//ul[@class='classifiedInfoList']/li[5]/span/text()").get()
oda_sayisi = response.xpath("//ul[@class='classifiedInfoList']/li[6]/span/text()").get()
bina_yasi = response.xpath("//ul[@class='classifiedInfoList']/li[7]/span/text()").get()
bulundugu_kat = response.xpath("//ul[@class='classifiedInfoList']/li[8]/span/text()").get()
kat_sayisi = response.xpath("//ul[@class='classifiedInfoList']/li[9]/span/text()").get()
isitma = response.xpath("//ul[@class='classifiedInfoList']/li[10]/span/text()").get()
banyo_sayisi = response.xpath("//ul[@class='classifiedInfoList']/li[11]/span/text()").get()
balkon = response.xpath("//ul[@class='classifiedInfoList']/li[12]/span/text()").get()
esyali = response.xpath("//ul[@class='classifiedInfoList']/li[13]/span/text()").get()
kullanim_durumu = response.xpath("//ul[@class='classifiedInfoList']/li[14]/span/text()").get()
site_icerisinde = response.xpath("//ul[@class='classifiedInfoList']/li[15]/span/text()").get()
site_adi = response.xpath("//ul[@class='classifiedInfoList']/li[16]/span/text()").get()
aidat = response.xpath("//ul[@class='classifiedInfoList']/li[17]/span/text()").get()
krediye_uygun = response.xpath("//ul[@class='classifiedInfoList']/li[18]/span/text()").get()
tapu_durumu = response.xpath("//ul[@class='classifiedInfoList']/li[19]/span/text()").get()
kimden = response.xpath("//ul[@class='classifiedInfoList']/li[20]/span/text()").get()
fiyat = response.xpath("//div[@class='classifiedInfo ']/h3[contains(text(), 'TL')]/text()").get()
box = response.xpath("//div[@class='uiBoxContainer classifiedDescription']/ul/li[@class='selected']/text()").getall()
for word in box:
box = word.replace(u"n", "")
box = word.strip()
set_box =set({box})
#print(set_box)

ozellikler = set_box
item = {
'ilan_no' : ilan_no.strip(),
'fiyat' : fiyat.strip(),
'kimden' : kimden.strip(),
'tapu_durumu' : tapu_durumu.strip(),
'krediye_uygun' : krediye_uygun.strip(),
'aidat' : aidat.strip(),
'site_adi' : site_adi.strip(),
'site_icerisinde' : site_icerisinde.strip(),
'kullanim_durumu' : kullanim_durumu.strip(),
'esyali' : esyali.strip(),
'balkon' : balkon.strip(),
'banyo_sayisi' : banyo_sayisi.strip(),
'isitma' : isitma.strip(),
'kat_sayisi' : kat_sayisi.strip(),
'bulundugu_kat' : bulundugu_kat.strip(),
'bina_yasi' : bina_yasi.strip(),
'oda_sayisi' : oda_sayisi.strip(),
'metrekare_net' : metrekare_net.strip(),
'metrekare_brut' : metrekare_brut.strip(),
'emlak_tipi' : emlak_tipi.strip(),
'ilan_tarihi' : ilan_tarihi.strip(),

'ozellikler' : set_box.getall(),
}
print(item)

这是我的密码。当我点击回车键时，它会给我AttributeError

'ozellikler' : set_box.getall(),
AttributeError: 'set' object has no attribute 'getall'

然后，我从相关变量中删除了getall；set_box"；。所以我的代码变成了：

import scrapy
from urllib.parse import urljoin
from scrapy import signals
from pydispatch import dispatcher
import json
import time
class AtasehirSpider(scrapy.Spider):
name = 'atasehir'
allowed_domains = ['www.sahibinden.com','sahibinden.com']
start_urls = ['https://www.sahibinden.com/satilik-daire/istanbul-atasehir-ornek-ornek-mh.?address_region=2&sorting=price_asc&price_min=805000&price_max=900000']
def parse(self, response):
for ad in response.xpath("//td[@class='searchResultsLargeThumbnail']/a/@href").getall():
time.sleep(10)
ads = response.urljoin(ad)
yield response.follow(url=ads, callback=self.parseInnerPage)


next_page_url = response.xpath("//ul[@class='pageNaviButtons']/li/a[@title='Sonraki']/@href").extract_first()
nextPage = response.urljoin(next_page_url)
if nextPage is not None:
time.sleep(10)
yield scrapy.Request(nextPage)

def parseInnerPage(self, response):
ilan_no = response.xpath("//ul[@class='classifiedInfoList']/li[1]/span/text()").get()
ilan_tarihi = response.xpath("//ul[@class='classifiedInfoList']/li[2]/span/text()").get()
emlak_tipi = response.xpath("//ul[@class='classifiedInfoList']/li[3]/span/text()").get()
metrekare_brut = response.xpath("//ul[@class='classifiedInfoList']/li[4]/span/text()").get()
metrekare_net = response.xpath("//ul[@class='classifiedInfoList']/li[5]/span/text()").get()
oda_sayisi = response.xpath("//ul[@class='classifiedInfoList']/li[6]/span/text()").get()
bina_yasi = response.xpath("//ul[@class='classifiedInfoList']/li[7]/span/text()").get()
bulundugu_kat = response.xpath("//ul[@class='classifiedInfoList']/li[8]/span/text()").get()
kat_sayisi = response.xpath("//ul[@class='classifiedInfoList']/li[9]/span/text()").get()
isitma = response.xpath("//ul[@class='classifiedInfoList']/li[10]/span/text()").get()
banyo_sayisi = response.xpath("//ul[@class='classifiedInfoList']/li[11]/span/text()").get()
balkon = response.xpath("//ul[@class='classifiedInfoList']/li[12]/span/text()").get()
esyali = response.xpath("//ul[@class='classifiedInfoList']/li[13]/span/text()").get()
kullanim_durumu = response.xpath("//ul[@class='classifiedInfoList']/li[14]/span/text()").get()
site_icerisinde = response.xpath("//ul[@class='classifiedInfoList']/li[15]/span/text()").get()
site_adi = response.xpath("//ul[@class='classifiedInfoList']/li[16]/span/text()").get()
aidat = response.xpath("//ul[@class='classifiedInfoList']/li[17]/span/text()").get()
krediye_uygun = response.xpath("//ul[@class='classifiedInfoList']/li[18]/span/text()").get()
tapu_durumu = response.xpath("//ul[@class='classifiedInfoList']/li[19]/span/text()").get()
kimden = response.xpath("//ul[@class='classifiedInfoList']/li[20]/span/text()").get()
fiyat = response.xpath("//div[@class='classifiedInfo ']/h3[contains(text(), 'TL')]/text()").get()
box = response.xpath("//div[@class='uiBoxContainer classifiedDescription']/ul/li[@class='selected']/text()").getall()
for word in box:
box = word.replace(u"n", "")
box = word.strip()
set_box =set({box})
#print(set_box)

ozellikler = set_box
item = {
'ilan_no' : ilan_no.strip(),
'fiyat' : fiyat.strip(),
'kimden' : kimden.strip(),
'tapu_durumu' : tapu_durumu.strip(),
'krediye_uygun' : krediye_uygun.strip(),
'aidat' : aidat.strip(),
'site_adi' : site_adi.strip(),
'site_icerisinde' : site_icerisinde.strip(),
'kullanim_durumu' : kullanim_durumu.strip(),
'esyali' : esyali.strip(),
'balkon' : balkon.strip(),
'banyo_sayisi' : banyo_sayisi.strip(),
'isitma' : isitma.strip(),
'kat_sayisi' : kat_sayisi.strip(),
'bulundugu_kat' : bulundugu_kat.strip(),
'bina_yasi' : bina_yasi.strip(),
'oda_sayisi' : oda_sayisi.strip(),
'metrekare_net' : metrekare_net.strip(),
'metrekare_brut' : metrekare_brut.strip(),
'emlak_tipi' : emlak_tipi.strip(),
'ilan_tarihi' : ilan_tarihi.strip(),
'ozellikler' : set_box
}
print(item)

这是我的回报：

{'ilan_no': '1028261219', 'fiyat': '870.000 TL', 'kimden': 'Emlak Ofisinden', 'tapu_durumu': 'Kat Mülkiyetli', 'krediye_uygun': 'Evet', 'aidat': 'Belirtilmemiş', 'site_adi': 'Belirtilmemiş', 'site_icerisinde': 'Hayır', 'kullanim_durumu': 'Kiracılı', 'esyali': 'Evet', 'balkon': 'Var', 'banyo_sayisi': '1', 'isitma': 'Doğalgaz (Kombi)', 'kat_sayisi': '3', 'bulundugu_kat': 'Bahçe Katı', 'bina_yasi': '5-10 arası', 'oda_sayisi': '1+1', 'metrekare_net': '50', 'metrekare_brut': '60', 'emlak_tipi': 'Satılık Daire', 'ilan_tarihi': '28 Eylül 2022', 'ozellikler': {'Bahçeli'}}

"ozellikler"；变量应包括50个以上的项目。但其中只有一个(最后一个(似乎是这里的回归。我打印了set_box，这里是"；ozellikler"；变量：

{'Güney'}
{'Kuzey'}
{'Amerikan Kapı'}
{'Amerikan Mutfak'}
{'Beyaz Eşya'}
{'Buzdolabı'}
{'Çamaşır Makinesi'}
{'Çelik Kapı'}
{'Duşakabin'}
{'Isıcam'}
{'Kartonpiyer'}
{'Laminat Zemin'}
{'Mutfak (Laminat)'}
{'Mutfak Doğalgazı'}
{'PVC Doğrama'}
{'Seramik Zemin'}
{'Set Üstü Ocak'}
{'Isı Yalıtımı'}
{'Ses Yalıtımı'}
{'Uydu'}
{'Alışveriş Merkezi'}
{'Belediye'}
{'Cami'}
{'Cemevi'}
{'Eczane'}
{'Hastane'}
{'İlkokul-Ortaokul'}
{'İtfaiye'}
{'Lise'}
{'Market'}
{'Park'}
{'Polis Merkezi'}
{'Sağlık Ocağı'}
{'Semt Pazarı'}
{'Spor Salonu'}
{'Şehir Merkezi'}
{'Üniversite'}
{'Anayol'}
{'Avrasya Tüneli'}
{'Boğaz Köprüleri'}
{'Cadde'}
{'Dolmuş'}
{'E-5'}
{'Marmaray'}
{'Metro'}
{'Metrobüs'}
{'Minibüs'}
{'Otobüs Durağı'}
{'TEM'}
{'Doğa'}
{'Şehir'}
{'Bahçe Katı'}
{'Bahçeli'}

我怎样才能拿到所有的物品？

Answer 1

该网站似乎不允许来自国外的访问，但我认为问题在于如何解析box变量。

尝试用以下内容替换set_box循环：

...
box = response.xpath("//div[@class='uiBoxContainer classifiedDescription']/ul/li[@class='selected']/text()").getall()
set_box = set()
for word in box:
word = word.replace("n", "").strip()
set_box.add(word)
...
...