HttpRequester
The default (ready-to-use) HTTP requester.
using AngleSharp.Extensions;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
namespace AngleSharp.Network.Default
{
public sealed class HttpRequester : IRequester
{
private sealed class RequestState
{
private readonly CookieContainer ;
private readonly IDictionary<string, string> _headers;
private readonly HttpWebRequest _http;
private readonly IRequest _request;
private readonly byte[] _buffer;
public RequestState(IRequest request, IDictionary<string, string> headers, Action<HttpWebRequest> setup)
{
_cookies = new CookieContainer();
_headers = headers;
_request = request;
_http = (WebRequest.Create(request.Address) as HttpWebRequest);
_http.CookieContainer = _cookies;
_http.Method = request.Method.ToString().ToUpperInvariant();
_buffer = new byte[4096];
SetHeaders();
SetCookies();
AllowCompression();
DisableAutoRedirect();
setup(_http);
}
public async Task<IResponse> RequestAsync(CancellationToken cancellationToken)
{
HttpWebRequest http = _http;
cancellationToken.Register(((WebRequest)http).Abort);
if (_request.Method == HttpMethod.Post || _request.Method == HttpMethod.Put) {
TaskFactory factory = Task.Factory;
HttpWebRequest http2 = _http;
Func<AsyncCallback, object, IAsyncResult> beginMethod = ((WebRequest)http2).BeginGetRequestStream;
HttpWebRequest http3 = _http;
SendRequest(await factory.FromAsync(beginMethod, (Func<IAsyncResult, Stream>)((WebRequest)http3).EndGetRequestStream, (object)null).ConfigureAwait(false));
}
WebResponse webResponse;
try {
TaskFactory factory2 = Task.Factory;
HttpWebRequest http4 = _http;
Func<AsyncCallback, object, IAsyncResult> beginMethod2 = ((WebRequest)http4).BeginGetResponse;
HttpWebRequest http5 = _http;
webResponse = await factory2.FromAsync(beginMethod2, (Func<IAsyncResult, WebResponse>)((WebRequest)http5).EndGetResponse, (object)null).ConfigureAwait(false);
} catch (WebException ex) {
webResponse = ex.Response;
}
RaiseConnectionLimit(_http);
return GetResponse(webResponse as HttpWebResponse);
}
private void SendRequest(Stream target)
{
Stream content = _request.Content;
while (content != null) {
int num = content.Read(_buffer, 0, 4096);
if (num == 0)
break;
target.Write(_buffer, 0, num);
}
}
private Response GetResponse(HttpWebResponse response)
{
if (response != null) {
CookieCollection cookies = _cookies.GetCookies(response.ResponseUri);
var enumerable = from m in response.Headers.AllKeys
select new {
Key = m,
Value = response.Headers.get_Item(m)
};
Response response2 = new Response {
Content = response.GetResponseStream(),
StatusCode = response.StatusCode,
Address = Url.Convert(response.ResponseUri)
};
foreach (var item in enumerable) {
response2.Headers.Add(item.Key, item.Value);
}
if (cookies.Count > 0) {
IEnumerable<string> values = from m in cookies.OfType<Cookie>()
select m.ToString();
response2.Headers[HeaderNames.SetCookie] = string.Join(", ", values);
}
return response2;
}
return null;
}
private void AddHeader(string key, string value)
{
if (key.Is(HeaderNames.Accept))
_http.Accept = value;
else if (key.Is(HeaderNames.ContentType)) {
_http.ContentType = value;
} else if (key.Is(HeaderNames.Expect)) {
SetProperty(HeaderNames.Expect, value);
} else if (key.Is(HeaderNames.Date)) {
SetProperty(HeaderNames.Date, DateTime.Parse(value));
} else if (key.Is(HeaderNames.Host)) {
SetProperty(HeaderNames.Host, value);
} else if (key.Is(HeaderNames.IfModifiedSince)) {
SetProperty("IfModifiedSince", DateTime.Parse(value));
} else if (key.Is(HeaderNames.Referer)) {
SetProperty(HeaderNames.Referer, value);
} else if (key.Is(HeaderNames.UserAgent)) {
SetProperty("UserAgent", value);
} else if (!key.Is(HeaderNames.Connection) && !key.Is(HeaderNames.Range) && !key.Is(HeaderNames.ContentLength) && !key.Is(HeaderNames.TransferEncoding)) {
_http.Headers.set_Item(key, value);
}
}
private void SetCookies()
{
string orDefault = _request.Headers.GetOrDefault(HeaderNames.Cookie, string.Empty);
_cookies.SetCookies(_http.RequestUri, orDefault.Replace(';', ','));
}
private void SetHeaders()
{
foreach (KeyValuePair<string, string> header in _headers) {
AddHeader(header.Key, header.Value);
}
foreach (KeyValuePair<string, string> header2 in _request.Headers) {
AddHeader(header2.Key, header2.Value);
}
}
private void AllowCompression()
{
SetProperty("AutomaticDecompression", 3);
}
private void DisableAutoRedirect()
{
SetProperty("AllowAutoRedirect", false);
}
private void SetProperty(string name, object value)
{
PropertyInfo value2 = null;
if (!PropCache.TryGetValue(name, out value2)) {
lock (PropCache) {
if (!PropCache.TryGetValue(name, out value2)) {
value2 = PortableExtensions.GetProperty(_http.GetType(), name);
PropCache.Add(name, value2);
}
}
}
if (!Restricted.Contains(name) && (object)value2 != null && value2.CanWrite)
try {
value2.SetValue(_http, value, null);
} catch {
lock (Restricted) {
if (!Restricted.Contains(name))
Restricted.Add(name);
}
}
}
}
private const int BufferSize = 4096;
private static readonly string Version = typeof(HttpRequester).GetTypeInfo().get_Assembly().GetCustomAttribute<AssemblyFileVersionAttribute>()
.Version;
private static readonly string AgentName = "AngleSharp/" + Version;
private static readonly Dictionary<string, PropertyInfo> PropCache = new Dictionary<string, PropertyInfo>();
private static readonly List<string> Restricted = new List<string>();
private TimeSpan _timeOut;
private readonly Action<HttpWebRequest> _setup;
private readonly Dictionary<string, string> _headers;
public IDictionary<string, string> Headers => _headers;
public TimeSpan Timeout {
get {
return _timeOut;
}
set {
_timeOut = value;
}
}
public HttpRequester(string userAgent = null, Action<HttpWebRequest> setup = null)
{
_timeOut = new TimeSpan(0, 0, 0, 45);
_setup = (setup ?? ((Action<HttpWebRequest>)delegate {
}));
_headers = new Dictionary<string, string> {
{
HeaderNames.UserAgent,
userAgent ?? AgentName
}
};
}
public bool SupportsProtocol(string protocol)
{
return protocol.IsOneOf(ProtocolNames.Http, ProtocolNames.Https);
}
public async Task<IResponse> RequestAsync(IRequest request, CancellationToken cancellationToken)
{
CancellationTokenSource cancellationTokenSource = CreateTimeoutToken(_timeOut);
RequestState requestState = new RequestState(request, _headers, _setup);
using (cancellationToken.Register(cancellationTokenSource.Cancel))
return await requestState.RequestAsync(cancellationTokenSource.Token).ConfigureAwait(false);
}
private static CancellationTokenSource CreateTimeoutToken(TimeSpan elapsed)
{
return new CancellationTokenSource(elapsed);
}
private static void RaiseConnectionLimit(HttpWebRequest http)
{
object obj = typeof(HttpWebRequest).GetField("_ServicePoint")?.GetValue(http);
if (obj != null)
PortableExtensions.GetProperty(obj.GetType(), "ConnectionLimit")?.SetValue(obj, 1024, null);
}
}
}