爬取搜狗微信文章

一、获取索引页。
添加好headers，params直接get请求即可。
二、在索引页中获取详情页的url。

在浏览器中直接打开这个url，仔细看浏览器上方中url，你会发现url会跳转，就是说他进行了重定向。这里我们需要使用fiddler这个抓包工具对他进行抓包。

你会发现他返回的是一段js代码，代码拼接成的url就是我们想要的真正的real_url，这里我们直接用正则表达式，进行匹配、拼接就ok了。

三、获取详情页。
既然获取到了详情页的真实url，剩下的构造一下请求头headers，直接请求即可，这里我不推荐加cookie参数到请求头里，因为加上之后访问次数过多，他会封cookie，让你输入验证码进行验证。
四、总结。
我看网上说他是封ip，封cookie的，但是我爬了几百篇并没有发现这个情况，之后遇到再补充。附上代码：

import requests
from lxml import etree
import re
import urllib3

# 重定向、封ip、封cookie


urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

def get_index(start_url, page):
    params = {
        'query': '美女',
        '_sug_type_': '',
        'sut': '1828',
        'lkt': '1,1597298703444,1597298703444',
        's_from': 'input',
        '_sug_': 'y',
        'type': '2',
        'sst0': '1597298703546',
        'page': page,
        'ie': 'utf8',
        'w': '01019900',
        'dr': '1',
    }
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Cookie': 'ABTEST=5|1597380273|v1; SNUID=59BBCE9AEAEF4626F956F592EA96FE70; IPLOC=CN3709; SUID=B3512470160DA00A000000005F3616B1; JSESSIONID=aaapSleD9Qep4fAnwHYox; SUV=0004E05A702451B35F3616B1A47C7070; SUID=B35124706A13A00A000000005F3616B1',
        'Host': 'weixin.sogou.com',
        'Referer': 'https://weixin.sogou.com/weixin?query=%E7%BE%8E%E5%A5%B3&_sug_type_=&sut=1828&lkt=1%2C1597298703444%2C1597298703444&s_from=input&_sug_=y&type=2&sst0=1597298703546&page=4&ie=utf8&w=01019900&dr=1',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-User': '?1',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36',
    }
    response = requests.get(url=start_url, params=params, headers=headers, verify=False)

    if response.status_code == 200:
        html = etree.HTML(response.text)
        urls = html.xpath('//ul[@class="news-list"]/li//h3/a/@href')
        for url in urls:
            url = 'https://weixin.sogou.com/' + url
            print(url)
            yield url
    else:
        print('getting index page fail')


def get_real_url(url):
    headers = {
        'Host': 'weixin.sogou.com',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-User': '?1',
        'Sec-Fetch-Dest': 'document',
        'Referer': 'https://weixin.sogou.com/weixin?type=2&s_from=input&query=%E7%BE%8E%E5%A5%B3&ie=utf8&_sug_=n&_sug_type_=',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
        'Cookie': 'SUID=D85024704A21A00A000000005EFDF58C; SUV=0035E9CD702450D85EFDF58D4E7EB415; weixinIndexVisited=1; LCLKINT=121; LSTMV=468%2C24; pgv_pvi=2192807936; ABTEST=0|1596950375|v1; IPLOC=CN3709; SNUID=C32054007175DCCBC54D7C1071164832; JSESSIONID=aaaMv1RDeodhj5yAwCYox; PHPSESSID=34n87c5popss5ckq1gcinpc9s2',
    }
    response = requests.get(url=url, headers=headers, verify=False)
    pattern = re.compile('\+=.*?\'(.*?)\';', re.S)
    url_list = re.findall(pattern, response.text)
    real_url = ''
    for i in url_list:
        real_url += i
    real_url.replace('@', '')
    return real_url


def get_detail(url):
    headers = {
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
        'cache-control': 'max-age=0',
        'if-modified-since': 'Fri, 14 Aug 2020 12:33:50 +0800',
        'sec-fetch-dest': 'document',
        'sec-fetch-mode': 'navigate',
        'sec-fetch-site': 'cross-site',
        'sec-fetch-user': '?1',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36',
    }
    response = requests.get(url, headers=headers,verify=False)

    html = etree.HTML(response.text)
    if html.xpath('//*[@id="activity-name"]/text()'):
        title = html.xpath('//*[@id="activity-name"]/text()')[0].strip()
    else:
        title = ''
    result = {
        'title': title,
    }
    return result


if __name__ == '__main__':
    start_url = 'https://weixin.sogou.com/weixin'
    for page in range(1, 11):
        urls = get_index(start_url=start_url, page=page)
        for url in urls:
            real_url = get_real_url(url)
            result = get_detail(real_url)
            print(result)