HttpRequester
The default (ready-to-use) HTTP requester.
using AngleSharp.Extensions;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
namespace AngleSharp.Network.Default
{
public sealed class HttpRequester : IRequester
{
private sealed class RequestState
{
private TaskCompletionSource<bool> _completed;
private HttpWebResponse _response;
private readonly CookieContainer ;
private readonly HttpWebRequest _http;
private readonly IRequest _request;
private readonly byte[] _buffer;
public RequestState(IRequest request, IDictionary<string, string> headers)
{
string orDefault = request.Headers.GetOrDefault(HeaderNames.Cookie, string.Empty);
_cookies = new CookieContainer();
_request = request;
_http = (WebRequest.Create(request.Address) as HttpWebRequest);
_http.CookieContainer = _cookies;
_http.Method = request.Method.ToString().ToUpperInvariant();
_buffer = new byte[4096];
_completed = new TaskCompletionSource<bool>();
foreach (KeyValuePair<string, string> header in headers) {
AddHeader(header.Key, header.Value);
}
foreach (KeyValuePair<string, string> header2 in request.Headers) {
AddHeader(header2.Key, header2.Value);
}
_cookies.SetCookies(_http.RequestUri, orDefault);
}
public async Task<IResponse> RequestAsync(CancellationToken cancellationToken)
{
if (_request.Method == HttpMethod.Post || _request.Method == HttpMethod.Put) {
_http.BeginGetRequestStream(SendRequest, _request);
if (cancellationToken.IsCancellationRequested)
return null;
await _completed.Task.ConfigureAwait(false);
_completed = new TaskCompletionSource<bool>();
}
if (cancellationToken.IsCancellationRequested)
return null;
_http.BeginGetResponse(ReceiveResponse, null);
await _completed.Task.ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
return null;
return GetResponse();
}
private void SendRequest(IAsyncResult ar)
{
IRequest request = (IRequest)ar.AsyncState;
Stream content = request.Content;
Stream stream = _http.EndGetRequestStream(ar);
if (content != null) {
while (content != null) {
int num = content.Read(_buffer, 0, 4096);
if (num == 0)
break;
stream.Write(_buffer, 0, num);
}
}
_completed.SetResult(true);
}
private void ReceiveResponse(IAsyncResult ar)
{
try {
_response = (HttpWebResponse)_http.EndGetResponse(ar);
} catch (WebException ex) {
_response = (HttpWebResponse)ex.Response;
}
_completed.SetResult(true);
}
private Response GetResponse()
{
if (_response == null)
return null;
Response response = new Response();
string cookieHeader = _cookies.GetCookieHeader(_response.ResponseUri);
var enumerable = from m in _response.Headers.AllKeys
select new {
Key = m,
Value = _response.Headers.get_Item(m)
};
response.Content = _response.GetResponseStream();
response.StatusCode = _response.StatusCode;
response.Address = Url.Convert(_response.ResponseUri);
foreach (var item in enumerable) {
response.Headers.Add(item.Key, item.Value);
}
if (cookieHeader != null)
response.Headers[HeaderNames.SetCookie] = cookieHeader;
return response;
}
private void AddHeader(string key, string value)
{
if (key == HeaderNames.Accept)
_http.Accept = value;
else if (key == HeaderNames.ContentType) {
_http.ContentType = value;
} else if (key == HeaderNames.Expect) {
SetProperty(HeaderNames.Expect, value);
} else if (key == HeaderNames.Date) {
SetProperty(HeaderNames.Date, DateTime.Parse(value));
} else if (key == HeaderNames.Host) {
SetProperty(HeaderNames.Host, value);
} else if (key == HeaderNames.IfModifiedSince) {
SetProperty("IfModifiedSince", DateTime.Parse(value));
} else if (key == HeaderNames.Referer) {
SetProperty(HeaderNames.Referer, value);
} else if (key == HeaderNames.UserAgent) {
SetProperty("UserAgent", value);
} else if (key != HeaderNames.Connection && key != HeaderNames.Range && key != HeaderNames.ContentLength && key != HeaderNames.TransferEncoding) {
_http.Headers.set_Item(key, value);
}
}
private void SetProperty(string name, object value)
{
if (!_propCache.ContainsKey(name))
_propCache.Add(name, _http.GetType().GetTypeInfo().GetDeclaredProperty(name));
PropertyInfo propertyInfo = _propCache[name];
if (!_restricted.Contains(name) && (object)propertyInfo != null && propertyInfo.CanWrite)
try {
propertyInfo.SetValue(_http, value, null);
} catch {
_restricted.Add(name);
}
}
}
private const int BufferSize = 4096;
private static readonly string _version;
private static readonly string _agentName;
private static readonly Dictionary<string, PropertyInfo> _propCache;
private static readonly List<string> _restricted;
private TimeSpan _timeOut;
private readonly Dictionary<string, string> _headers;
public Dictionary<string, string> Headers => _headers;
public TimeSpan Timeout {
get {
return _timeOut;
}
set {
_timeOut = value;
}
}
static HttpRequester()
{
_version = typeof(HttpRequester).GetTypeInfo().get_Assembly().GetCustomAttribute<AssemblyFileVersionAttribute>()
.Version;
_agentName = "AngleSharp/" + _version;
_propCache = new Dictionary<string, PropertyInfo>();
_restricted = new List<string>();
}
public HttpRequester(string userAgent = null)
{
_timeOut = new TimeSpan(0, 0, 0, 45);
_headers = new Dictionary<string, string>();
_headers.Add("User-Agent", userAgent ?? _agentName);
}
public bool SupportsProtocol(string protocol)
{
if (!(KnownProtocols.Http == protocol))
return KnownProtocols.Https == protocol;
return true;
}
public Task<IResponse> RequestAsync(IRequest request, CancellationToken cancellationToken)
{
RequestState requestState = new RequestState(request, _headers);
return requestState.RequestAsync(cancellationToken);
}
}
}