C#爬虫入门之简单GET请求-爬取起点小说

常用CSS选择器

名称	形式	例子	说明
标签寻址器	标签名1,标签名2…标签名n	pq('p,div')	选择html里的body里的p标签
后代选择器	父标签名 子标签名…曾孙标签名	pq('html body p')	选择class="demo"的所有标签
类选择器	.class属性值	pq('.demo')	选择id="first"的所有标签
id选择器	#id属性值	pq('#first')	选择所有标签
通用选择器	*	pq('*')	选择属性名为target的所有标签
属性选择器	[属性名] [属性名=属性值]	pq('[target]')	选择名称为<p>或<div>的的所有标签

异步爬取

/// <summary>
/// 获取字符串
/// </summary>
/// <param name="url">链接地址</param>
/// <param name="headers">头部信息</param>
/// <returns>返回字符串</returns>
public static async Task<string> GetStringAsync(string url,WebHeaderCollection headers)
{
    try
    {
        var re = WebRequest.Create(url) as HttpWebRequest;
        re.Method = "GET";
        re.Headers = headers;
        re.AutomaticDecompression = DecompressionMethods.All;
      //  re.Timeout = 120;
        var ans = await re.GetResponseAsync();
        using (StreamReader sr = new StreamReader(ans.GetResponseStream()))
        {
            return sr.ReadToEnd();
        }
    }
    catch(Exception ex)
    {
        WriteLine(ex.ToString());
        return "ERROR";
    }
}

提取小说内容

/// <summary>
/// 获取文本
/// </summary>
/// <param name="html">解析后的网页</param>
public static List<string> GetText(string html)
{
    List<string> ls = new List<string>();
    //解析
    var parser = new HtmlParser();
    var text = parser.ParseDocument(html);
    foreach (var item in text.QuerySelectorAll("div.text-wrap div[class] p"))
    {
        ls.Add(item.TextContent);
    }
    return ls;
}

提取下一页链接

/// <summary>
/// 获取下一页链接
/// </summary>
/// <param name="html"></param>
/// <returns>返回下一页链接</returns>
public static string GetNext(string html)
{
    try
    {
        //解析
        var parser = new HtmlParser();
        var text = parser.ParseDocument(html);
        var next = text.QuerySelector("a#j_chapterNext");
        return "https:" + next.Attributes["href"].Value;
    }
    catch(Exception ex)
    {
        WriteLine(ex);
        return "ERROR";
    }
}

写入文件

/// <summary>
/// 写入文件
/// </summary>
public static void ToFile(List<string> ls)
{
    foreach (var item in ls)
    {
        using(StreamWriter sw=new StreamWriter(path:@"data.txt",
            append:true,encoding:Encoding.UTF8))
        {
            sw.WriteLine(item);
        }
    }
}

Main方法

public static async Task Main(string[] args)
{
    //url链接
    var url = "https://read.qidian.com/chapter/7KON2BWNFT1aXyNuGKajlg2/s1G7TqJl3HRMs5iq0oQwLQ2";
    //伪装头部信息
    var headers = new WebHeaderCollection();
    headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
    //爬取网站
    while(url!="ERROR")
    {
        var html = await GetStringAsync(url, headers);
        var ls=GetText(html);
        ToFile(ls);
        url = GetNext(html);
    }
}