使用的是pycharm,爬虫
mport scrapy
from zuowen.items import ZuowenItem
class ZuoweneSpider(scrapy.Spider):
name = 'zuowene'
allowed_domains = ['https://www.zuowen.com']
start_urls = ['https://www.zuowen.com/xiaoxue/ernianji/']
def parse(self, response):
#写人作文网址获取
xieren = response.xpath("//div[@class='tczw_c']/div[1]/div[1]/ul/li[1]/a/@href").extract_first()
yield scrapy.Request(url=xieren,
callback=self.parse_url,
dont_filter=True)
#处理网址
def parse_url(self, response):
tr_list = response.xpath("//div[@class='artlist_l']/div")[1:-1]
for tr in tr_list:
item = ZuowenItem()
item["title"] = tr.xpath("./div[1]/a/text()").extract_first()
item["url"] = tr.xpath("./div[2]/a/@href").extract_first()
item["category"] = tr.xpath("//div[@class='artlist_l_t']/h1/a/text()").extract_first()
yield scrapy.Request(
item["url"],
callback=self.parse_detail,
meta={"item": item},
dont_filter=True
)
#翻页
next_ur1 = response.xpath("//div[@class='artpage']/a[text()='下一页']/@href").extract_first()
if next_ur1 is not None:
yield scrapy.Request(next_ur1,
callback=self.parse_url,
dont_filter=True
)
# 处理详情,抓取作文内容
def parse_detail(self, response):
item = response.meta["item"]
item["content"] = response.xpath("//div[@class='con_content']//text()").extract()
yield item
items
import scrapy
class ZuowenItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
#标题
title=scrapy.Field()
#网址
url=scrapy.Field()
#作文内容
content=scrapy.Field()
#分类
category=scrapy.Field()
剩下没有设置
已经试过将dont_filter=True改为Flase,但玩之后连内容都没有了