This is an example of the lengths that I have gone to in order to comply with guidelines posted by kevin, and if I can be bothered to comply with the kevin's guidelines, the least that kevin can do is specify those guidelines in clear and explicit language that cannot be misinterpreted by any reasonable person.
Soapi.Gateway
A crossplatform (.net/silverlight/windows phone 7/mono) request gateway with thread-safe throttling and caching. The async classes and the cache are not show, this is just to quell the masses.
read: this is an issue that will affect you (the other users), whether you recognize it now or not. I recognize it and am doing my part to shore up the dam, but I need some reciprocal cooperation from the stewards of this service.
//
// Project: SOAPI
// http://soapi.codeplex.com
// http://stackapps.com/questions/386
//
// Copyright 2010, Sky Sanders
// Licensed under the GPL Version 2 license.
// http://soapi.codeplex.com/license
//
// Date: June 26 2010
// API ver 0.9 rev 2010.06.24.002
//
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using System.Threading;
#if SILVERLIGHT
using ICSharpCode.SharpZipLib.GZip;
#endif
namespace Soapi
{
public class Gateway : IGateway
{
private readonly object _throttleLock = new object();
private readonly ManualResetEvent _throttleGate = new ManualResetEvent(false);
private DateTime _lastRequestTime = DateTime.MinValue;
/// <summary>
/// The minimum length of time to allow between requests.
/// </summary>
public int ThrottleInterval { get; set; }
/// <summary>
/// The user-agent header to apply to all requests.
/// </summary>
public string UserAgent { get; set; }
private IGatewayCache _cache;
/// <summary>
/// The IGatewayCache that services this gateway.
/// </summary>
public IGatewayCache Cache
{
get { return _cache; }
set { _cache = value; }
}
#if !SILVERLIGHT
/// <summary>
/// The System.Net.IWebProxy instance assigned to this gateway.
/// </summary>
public IWebProxy Proxy { get; set; }
#endif
/// <summary>
/// <para>This method first checks the cache for the specified url, if found the process results are returned immediately.</para>
/// <para>If a cached result is not found, the request is throttled as per the ThrottleInterval, and upon release, an AsyncResult is returned.</para>
/// <para>Upon successful completion of the request, the response text is passed to the requestHandler, which returns an instance of T.</para>
/// <para>This value is cached after the callback completes, allowing the callback to examine and modify the results before caching.</para>
/// </summary>
/// <typeparam name="T">The type that is returned by the requestHandler</typeparam>
/// <param name="url">The path of the request.</param>
/// <param name="cacheRule">
/// <para>The rule to apply when getting/setting from the gateway cache. </para>
/// <para>e.g. for StackOverflow API, this would be the route.
/// '/sites' should have a rule of 2 hours applied.</para>
/// </param>
/// <param name="bypassCache">whether to bypass the gateway's cache</param>
/// <param name="timeout">when to force a timeout error, in ms.</param>
/// <param name="callback">The function that handles the completion of the request</param>
/// <param name="requestHandler">The function that processes the request and
/// returns an instance of <typeparamref name="T"></typeparamref></param>
/// <returns></returns>
public GatewayAsyncResult<T> BeginGetResponse<T>(string url, string cacheRule, bool bypassCache, int? timeout,
GatewayAsyncCallback<T> callback,
GatewayRequestHandlerDelegate<T> requestHandler)
where T : class, new()
{
var state = new GatewayRequestState
{
CacheRule = cacheRule,
Url = url,
Timeout = timeout
};
// this will contain metrics for the request upon return
var result = new GatewayAsyncResult<T>(callback, state);
if (!bypassCache)
{
object cached;
lock (_cache)
{
cached = Cache.Get(url);
}
if (cached != null)
{
state.Cached = true;
result.Complete((T) cached, true);
return result; // <--- take the cache and exit early, no throttle charge
}
}
lock (_throttleLock)
{
// if the last request was within the throttle interval, lets
// just chill a bit and wait.
if (_lastRequestTime.AddMilliseconds(ThrottleInterval) > DateTime.Now)
{
var waitTime =
(int) (_lastRequestTime.AddMilliseconds(ThrottleInterval) - DateTime.Now).TotalMilliseconds;
_throttleGate.WaitOne(waitTime);
_throttleGate.Reset();
}
// make a not of the time so we can check the next request to come through this lock
double requestInterval = (DateTime.Now - _lastRequestTime).TotalMilliseconds;
state.RequestInterval = requestInterval;
_lastRequestTime = DateTime.Now;
}
HttpWebRequest request = GetWebRequest(url);
Exception getResponseException = null;
T processed = null;
var timeoutWait = new AutoResetEvent(false);
AsyncCallback innerCallback = ar =>
{
try
{
state.ResponseText = GetResponseText(request.EndGetResponse(ar));
if (requestHandler != null)
{
processed = requestHandler(state);
}
}
catch (Exception e)
{
getResponseException = e;
}
finally
{
timeoutWait.Set();
}
};
IAsyncResult asyncResult = request.BeginGetResponse(innerCallback, null);
if (asyncResult.CompletedSynchronously)
{
result.Complete(processed, true);
SetCacheItem(url, cacheRule, processed);
}
else
{
if (!timeoutWait.WaitOne(timeout ?? 60000))
{
result.HandleException(new TimeoutException("No response received in time."), false);
}
else
{
if (getResponseException != null)
{
result.HandleException(getResponseException, false);
}
else
{
result.Complete(processed, false);
SetCacheItem(url, cacheRule, processed);
}
}
}
return result;
}
private void SetCacheItem(string uri, string ruleKey, object data)
{
Cache.Set(uri, ruleKey, data);
}
private HttpWebRequest GetWebRequest(string uri)
{
var webRequest = (HttpWebRequest) WebRequest.Create(uri);
#if !SILVERLIGHT
webRequest.Method = "GET";
webRequest.UserAgent = UserAgent;
webRequest.AutomaticDecompression = DecompressionMethods.GZip;
webRequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip");
if (Proxy != null)
{
webRequest.Proxy = Proxy;
}
#endif
return webRequest;
}
private static string GetResponseText(WebResponse response)
{
#if SILVERLIGHT
// silverlight platforms vary in implementations
// especially if ClientHttp is enabled so we will just
// be safe and check every request.
using (var responseStream = response.GetResponseStream())
{
var data = ReadFully(responseStream);
string text;
// check for magic numbers
if (data.Length > 2 && (data[0] == 31 && data[1] == 139))
{
using (var ms = new MemoryStream(data))
using (var gzip = new GZipInputStream(ms))
using (var reader = new StreamReader(gzip))
{
text = reader.ReadToEnd();
}
}
else
{
text = Encoding.UTF8.GetString(data, 0, data.Length);
}
return text;
}
#else
// Framework requests are always decompressed by the HttpResponse
using (Stream stream = response.GetResponseStream())
{
return Encoding.UTF8.GetString(ReadFully(stream));
}
#endif
}
private static byte[] ReadFully(Stream stream)
{
var buffer = new byte[32768];
using (var ms = new MemoryStream())
{
while (true)
{
int read = stream.Read(buffer, 0, buffer.Length);
if (read <= 0)
{
return ms.ToArray();
}
ms.Write(buffer, 0, read);
}
}
}
}
}
This is an actual implementation of what is required to comply with throttle and caching guidelines.
You see, it is not a trivial task, and to require that someone undertake such a task without unambiguous specifications is unreasonable at best.
