A coworkers wanted to learn how to do basic web scraping – for example finding all the <A HREF> links on a webpage. Naturally I directed him to the Html Agility Pack. He wanted a .NET Framework solution, otherwise I would have recommended Python's Beautiful Soup library.
To get him started and to show how simple it was, I provided the following demo code. It doesn't do much, but will give anyone a quick start.
using System;
using System.Linq;
using HAP=HtmlAgilityPack;
namespace DemoHtmlAgilityPack
{
class Program
{
private static void Main(string[] args)
{
using (var client = new System.Net.WebClient())
{
var filename = System.IO.Path.GetTempFileName();
client.DownloadFile("http://python.org", filename);
var doc = new HAP.HtmlDocument();
doc.Load(filename);
var root = doc.DocumentNode;
var a_nodes = root.Descendants("a").ToList();
foreach (var a_node in a_nodes)
{
Console.WriteLine();
Console.WriteLine("LINK: {0}", a_node.GetAttributeValue("href",""));
Console.WriteLine("TEXT: {0}", a_node.InnerText.Trim());
}
}
Console.ReadKey();
}
}
}
Not hard at all!