ZahiZou ZahiZou - 3 months ago 22
Python Question

in scrapy,i use xpath pick up html got many unnecessary "" and ,?

the website link : http://so.gushiwen.org/view_20788.aspx

this is what i want:



"detail_text": ["
寥落古行宫,宫花寂寞红。白头宫女在,闲坐说玄宗。
"],



but I got this :
"detail_text":
["
", "
", "
", "
", "
寥落古行宫,宫花寂寞红。", "白头宫女在,闲坐说玄宗。
"],



and this is my code :

#spider
class Tangshi3Spide(scrapy.Spider):
name = "tangshi3"
allowed_domains = ["gushiwen.org"]
start_urls = [
"http://so.gushiwen.org/view_20788.aspx"
]
def __init__(self):
self.items = []

def parse(self, response):
sel = Selector(response)
sites = sel.xpath('//div[@class="main3"]/div[@class="shileft"]')
domain = 'http://so.gushiwen.org'
for site in sites:
item = Tangshi3Item()
item['detail_title'] = site.xpath('div[@class="son1"]/h1/text()').extract()
item['detail_dynasty'] = site.xpath(
u'div[@class="son2"]/p/span[contains(text(),"朝代:")]/parent::p/text()').extract()
item['detail_translate_note_url'] = site.xpath('div[@id="fanyiShort676"]/p/a/u/parent::a/@href').extract()
item['detail_appreciation_url'] = site.xpath('div[@id="shangxiShort787"]/p/a/u/parent::a/@href').extract()
item['detail_background_url'] = site.xpath('div[@id="shangxiShort24492"]/p/a/u/parent::a/@href').extract()
#question line
item['detail_text'] = site.xpath('div[@class="son2"]/text()').extract()
self.items.append(item)
return self.items



#pipeline
class Tangshi3Pipeline(object):
def __init__(self):
self.file = codecs.open('tangshi_detail.json', 'w', encoding='utf-8')

def process_item(self, item, spider):
line = json.dumps(dict(item))
self.file.write(line.decode("unicode_escape"))
return item


how can I get the right text?

Answer

You can add predicate [normalize-space()] to avoid picking up empty text nodes i.e those containing whitespaces only :

item['detail_text'] = site.xpath('div[@class="son2"]/text()[normalize-space()]').extract()