公司編輯妹子需要爬取網頁內容，叫我幫忙做了一簡單的爬取工具
這是爬取網頁內容，像是這對大家來說都是不難得，但是在這裡有一些小改動，代碼獻上，大家參考
1 private string GetHttpWebRequest(string url)
2 {
3 HttpWebResponse result;
4 string strHTML = string.Empty;
5 try
6 {
7 Uri uri = new Uri(url);
8 WebRequest webReq = WebRequest.Create(uri);
9 WebResponse webRes = webReq.GetResponse();
10
11 HttpWebRequest myReq = (HttpWebRequest)webReq;
12 myReq.UserAgent = "User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
13 myReq.Accept = "*/*";
14 myReq.KeepAlive = true;
15 myReq.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
16 result = (HttpWebResponse)myReq.GetResponse();
17 Stream receviceStream = result.GetResponseStream();
18 StreamReader readerOfStream = new StreamReader(receviceStream, System.Text.Encoding.GetEncoding("utf-8"));
19 strHTML = readerOfStream.ReadToEnd();
20 readerOfStream.Close();
21 receviceStream.Close();
22 result.Close();
23 }
24 catch
25 {
26 Uri uri = new Uri(url);
27 WebRequest webReq = WebRequest.Create(uri);
28 HttpWebRequest myReq = (HttpWebRequest)webReq;
29 myReq.UserAgent = "User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
30 myReq.Accept = "*/*";
31 myReq.KeepAlive = true;
32 myReq.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
33 //result = (HttpWebResponse)myReq.GetResponse();
34 try
35 {
36 result = (HttpWebResponse)myReq.GetResponse();
37 }
38 catch (WebException ex)
39 {
40 result = (HttpWebResponse)ex.Response;
41 }
42 Stream receviceStream = result.GetResponseStream();
43 StreamReader readerOfStream = new StreamReader(receviceStream, System.Text.Encoding.GetEncoding("gb2312"));
44 strHTML = readerOfStream.ReadToEnd();
45 readerOfStream.Close();
46 receviceStream.Close();
47 result.Close();
48 }
49 return strHTML;
50 }
這是根據url爬取網頁遠嗎，有一些小改動，很多網頁有不同的編碼格式，甚至有些網站做了反爬取的防范，這個方法經過能夠改動也能爬去
以下是爬取網頁所有的網址鏈接
/// <summary>
/// 提取HTML代碼中的網址
/// </summary>
/// <param name="htmlCode"></param>
/// <returns></returns>
private static List<string> GetHyperLinks(string htmlCode, string url)
{
ArrayList al = new ArrayList();
bool IsGenxin = false;
StringBuilder weburlSB = new StringBuilder();//SQL
StringBuilder linkSb = new StringBuilder();//展示數據
List<string> Weburllistzx = new List<string>();//新增
List<string> Weburllist = new List<string>();//舊的
string ProductionContent = htmlCode;
Regex reg = new Regex(@"http(s)?://([\w-]+\.)+[\w-]+/?");
string wangzhanyuming = reg.Match(url, 0).Value;
MatchCollection mc = Regex.Matches(ProductionContent.Replace("href=\"/", "href=\"" + wangzhanyuming).Replace("href='/", "href='" + wangzhanyuming).Replace("href=/", "href=" + wangzhanyuming).Replace("href=\"./", "href=\"" + wangzhanyuming), @"<[aA][^>]* href=[^>]*>", RegexOptions.Singleline);
int Index = 1;
foreach (Match m in mc)
{
MatchCollection mc1 = Regex.Matches(m.Value, @"[a-zA-z]+://[^\s]*", RegexOptions.Singleline);
if (mc1.Count > 0)
{
foreach (Match m1 in mc1)
{
string linkurlstr = string.Empty;
linkurlstr = m1.Value.Replace("\"", "").Replace("'", "").Replace(">", "").Replace(";", "");
weburlSB.Append("$-$");
weburlSB.Append(linkurlstr);
weburlSB.Append("$_$");
if (!Weburllist.Contains(linkurlstr) && !Weburllistzx.Contains(linkurlstr))
{
IsGenxin = true;
Weburllistzx.Add(linkurlstr);
linkSb.AppendFormat("{0}<br/>", linkurlstr);
}
}
}
else
{
if (m.Value.IndexOf("javascript") == -1)
{
string amstr = string.Empty;
string wangzhanxiangduilujin = string.Empty;
wangzhanxiangduilujin = url.Substring(0, url.LastIndexOf("/") + 1);
amstr = m.Value.Replace("href=\"", "href=\"" + wangzhanxiangduilujin).Replace("href='", "href='" + wangzhanxiangduilujin);
MatchCollection mc11 = Regex.Matches(amstr, @"[a-zA-z]+://[^\s]*", RegexOptions.Singleline);
foreach (Match m1 in mc11)
{
string linkurlstr = string.Empty;
linkurlstr = m1.Value.Replace("\"", "").Replace("'", "").Replace(">", "").Replace(";", "");
weburlSB.Append("$-$");
weburlSB.Append(linkurlstr);
weburlSB.Append("$_$");
if (!Weburllist.Contains(linkurlstr) && !Weburllistzx.Contains(linkurlstr))
{
IsGenxin = true;
Weburllistzx.Add(linkurlstr);
linkSb.AppendFormat("{0}<br/>", linkurlstr);
}
}
}
}
Index++;
}
return Weburllistzx;
}
這塊的技術其實就是簡單的使用了正則去匹配！接下來獻上獲取標題，以及存儲到xml文件的方法
1 /// <summary>
2 /// // 把網址寫入xml文件
3 /// </summary>
4 /// <param name="strURL"></param>
5 /// <param name="alHyperLinks"></param>
6 private static void WriteToXml(string strURL, List<string> alHyperLinks)
7 {
8 XmlTextWriter writer = new XmlTextWriter(@"D:\HyperLinks.xml", Encoding.UTF8);
9 writer.Formatting = Formatting.Indented;
10 writer.WriteStartDocument(false);
11 writer.WriteDocType("HyperLinks", null, "urls.dtd", null);
12 writer.WriteComment("提取自" + strURL + "的超鏈接");
13 writer.WriteStartElement("HyperLinks");
14 writer.WriteStartElement("HyperLinks", null);
15 writer.WriteAttributeString("DateTime", DateTime.Now.ToString());
16 foreach (string str in alHyperLinks)
17 {
18 string title = GetDomain(str);
19 string body = str;
20 writer.WriteElementString(title, null, body);
21 }
22 writer.WriteEndElement();
23 writer.WriteEndElement();
24 writer.Flush();
25 writer.Close();
26 }
27 /// <summary>
28 /// 獲取網址的域名後綴
29 /// </summary>
30 /// <param name="strURL"></param>
31 /// <returns></returns>
32 private static string GetDomain(string strURL)
33 {
34 string retVal;
35 string strRegex = @"(\.com/|\.net/|\.cn/|\.org/|\.gov/)";
36 Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
37 Match m = r.Match(strURL);
38 retVal = m.ToString();
39 strRegex = @"\.|/$";
40 retVal = Regex.Replace(retVal, strRegex, "").ToString();
41 if (retVal == "")
42 retVal = "other";
43 return retVal;
44 }
45 /// <summary>
46 /// 獲取標題
47 /// </summary>
48 /// <param name="html"></param>
49 /// <returns></returns>
50 private static string GetTitle(string html)
51 {
52 string titleFilter = @"<title>[\s\S]*?</title>";
53 string h1Filter = @"<h1.*?>.*?</h1>";
54 string clearFilter = @"<.*?>";
55
56 string title = "";
57 Match match = Regex.Match(html, titleFilter, RegexOptions.IgnoreCase);
58 if (match.Success)
59 {
60 title = Regex.Replace(match.Groups[0].Value, clearFilter, "");
61 }
62
63 // 正文的標題一般在h1中，比title中的標題更干淨
64 match = Regex.Match(html, h1Filter, RegexOptions.IgnoreCase);
65 if (match.Success)
66 {
67 string h1 = Regex.Replace(match.Groups[0].Value, clearFilter, "");
68 if (!String.IsNullOrEmpty(h1) && title.StartsWith(h1))
69 {
70 title = h1;
71 }
72 }
73 return title;
74 }
這就是所用的全部方法，還是有很多需要改進之處！大家如果有發現不足之處還請指出，謝謝！
