用cookie刮擦动态页面

Question

我正在尝试将此页面刮掉一组Zipcodes。https://www.chase.com/mortgage/loan-findicer/search-results.html#action-search-search; zipcode-11747; lastName-; language-

如果将其放入浏览器中，则将获得结果，但是，在代码中尝试这样做。

首先我尝试了

HttpWebRequest  request = (HttpWebRequest )System.Net.WebRequest.Create(URI);
var sr = new System.IO.StreamReader(resp.GetResponseStream());
string page= sr.ReadToEnd().Trim();

但是，Fiddler中的插件生成的代码也无法正常工作。没有返回结果。那我到底想念什么？

private void MakeRequests()
{
    HttpWebResponse response;
    string responseText;
    if (Request_www_chase_com(out response))
    {
        responseText = ReadResponse(response);
        response.Close();
    }
}
private static string ReadResponse(HttpWebResponse response)
{
    using (Stream responseStream = response.GetResponseStream())
    {
        Stream streamToRead = responseStream;
        if (response.ContentEncoding.ToLower().Contains("gzip"))
        {
            streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress);
        }
        else if (response.ContentEncoding.ToLower().Contains("deflate"))
        {
            streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress);
        }
        using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8))
        {
            return streamReader.ReadToEnd();
        }
    }
}
private bool Request_www_chase_com(out HttpWebResponse response)
{
    response = null;
    try
    {
        HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://www.chase.com/mortgage/loan-officer/search-results.html");
        request.KeepAlive = true;
        request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
        request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
        request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36";
        request.Headers.Add("DNT", @"1");
        request.Referer = "https://mail.google.com/mail/u/0/?shva=1";
        request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip,deflate,sdch");
        request.Headers.Set(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8");
        request.Headers.Set(HttpRequestHeader.Cookie, @"v1st=3B46E5CCD302C2DE; marketlist=68|90|152|170|198; chasezip=zipcode=11577&county=Nassau&state=NY; ASP.NET_SessionId=kwybehscfioasswbl20wb14f; PC_1_0=n%3Dundefined|u%3Dundefined|l%3Dundefined|zip%3D11577|lastUpdate%3D2014-01-24|lastSent%3D2014-01-24|home%3Dpersonal|; SessionPersistence=CLICKSTREAMCLOUD%3A%3DvisitorId%3D%7CPROFILEDATA%3A%3D%7CSURFERINFO%3A%3Dbrowser%3DChrome%2COS%3DWindows%2Cresolution%3D1366x768%7C; fsr.s=%7B%22v2%22%3A-2%2C%22v1%22%3A1%2C%22rid%22%3A%22d464cf6-82273859-c860-572f-2944b%22%2C%22to%22%3A5%2C%22c%22%3A%22https%3A%2F%2Fwww.chase.com%2Fmortgage%2Floan-officer%2Fsearch-results.html%23action-search%3Bzipcode-11747%3Blastname-%3Blanguage-%22%2C%22pv%22%3A12%2C%22lc%22%3A%7B%22d18%22%3A%7B%22v%22%3A12%2C%22s%22%3Atrue%7D%7D%2C%22cd%22%3A18%2C%22sd%22%3A18%2C%22f%22%3A1390649574789%7D");
        request.IfModifiedSince = DateTime.Parse("Fri, 24 Jan 2014 20:18:51 GMT");
        response = (HttpWebResponse)request.GetResponse();
    }
    catch (WebException e)
    {
        if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response;
        else return false;
    }
    catch (Exception)
    {
        if (response != null) response.Close();
        return false;
    }
    return true;
}

Answer 1

要完成此工作，您需要解析HTML，然后下载并运行JavaScript。与其编写自己的浏览器，不如使用Web浏览器控件加载页面，然后刮擦其内部HTML。

Answer 2

页面使用ajax创建结果，因此您在响应中会看到的只是初始html