Python 爬取小说(斗破之无上)

极客都是这么看小说

• 代码为本人原创，且仅限于学习交流，请勿用于任何商业用途！本人不承担任何法律责任，如果涉及到侵权问题，请留言告知。

import requests
from bs4 import BeautifulSoup


class downloader(object):

    def __init__(self):
        self.base = 'http://m.yruan.com'
        self.article = '/article/53567/'
        self.page = '31977800_3.html'
        self.target = self.base + self.article + self.page

    def get_contents(self):
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                          'Chrome/70.0.3538.67 Safari/537.36',
        }
        params = {
            'kw': '赵丽颖吧',
            'pn': '50'
        }
        req = requests.get(url=self.target, headers=headers, params=params)
        # requests.post()
        req.encoding = 'utf-8'
        html = req.text
        bs = BeautifulSoup(html, 'html.parser')  # 得到一个 BeautifulSoup 的对象
        # print(bs.prettify())
        # print(bs.getText())
        novelcontent = bs.find(name='div', id="novelcontent")
        print(type(novelcontent))
        # print(novelcontent.find(name='p').text.replace('\n', ''))
        # print(novelcontent.find(name='p'))
        print(novelcontent.find(name='p').text.replace('\n\n', '\n'))
        # print(novelcontent.find(name='ul', class_='novelbutton'))
        next_a = novelcontent.find(name='ul', class_='novelbutton').find(name='p', class_='p1 p3').find(name='a')
        print(next_a.get('href'))
        # print(novelcontent)
        print("current_url: " + self.target)
        return self.base + next_a.get('href')


if __name__ == '__main__':
    dl = downloader()
    next_url = dl.get_contents()
    print("Next_url: " + next_url)