using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Net;
using Humanizer;
using IGDB;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using RestEase;
namespace gaseous_server.Classes.Metadata
{
///
/// Handles all metadata API communications
///
public class Communications
{
static Communications()
{
var handler = new HttpClientHandler();
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
client = new HttpClient(handler);
client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip");
client.DefaultRequestHeaders.Add("Accept-Encoding", "deflate");
}
private static IGDBClient igdb = new IGDBClient(
// Found in Twitch Developer portal for your app
Config.IGDB.ClientId,
Config.IGDB.Secret
);
private static HttpClient client = new HttpClient();
///
/// Configure metadata API communications
///
public static HasheousClient.Models.MetadataModel.MetadataSources MetadataSource
{
get
{
return _MetadataSource;
}
set
{
_MetadataSource = value;
switch (value)
{
case HasheousClient.Models.MetadataModel.MetadataSources.IGDB:
// set rate limiter avoidance values
RateLimitAvoidanceWait = 1500;
RateLimitAvoidanceThreshold = 3;
RateLimitAvoidancePeriod = 1;
// set rate limiter recovery values
RateLimitRecoveryWaitTime = 10000;
break;
default:
// leave all values at default
break;
}
}
}
private static HasheousClient.Models.MetadataModel.MetadataSources _MetadataSource = HasheousClient.Models.MetadataModel.MetadataSources.None;
// rate limit avoidance - what can we do to ensure that rate limiting is avoided?
// these values affect all communications
///
/// How long to wait to avoid hitting an API rate limiter
///
private static int RateLimitAvoidanceWait = 2000;
///
/// How many API calls in the period are allowed before we start introducing a wait
///
private static int RateLimitAvoidanceThreshold = 80;
///
/// A counter of API calls since the beginning of the period
///
private static int RateLimitAvoidanceCallCount = 0;
///
/// How large the period (in seconds) to measure API call counts against
///
private static int RateLimitAvoidancePeriod = 60;
///
/// The start of the rate limit avoidance period
///
private static DateTime RateLimitAvoidanceStartTime = DateTime.UtcNow;
///
/// Used to determine if we're already in rate limit avoidance mode - always query "InRateLimitAvoidanceMode"
/// for up to date mode status.
/// This bool is used to track status changes and should not be relied upon for current status.
///
private static bool InRateLimitAvoidanceModeStatus = false;
///
/// Determine if we're in rate limit avoidance mode.
///
private static bool InRateLimitAvoidanceMode
{
get
{
if (RateLimitAvoidanceStartTime.AddSeconds(RateLimitAvoidancePeriod) <= DateTime.UtcNow)
{
// avoidance period has expired - reset
RateLimitAvoidanceCallCount = 0;
RateLimitAvoidanceStartTime = DateTime.UtcNow;
return false;
}
else
{
// we're in the avoidance period
if (RateLimitAvoidanceCallCount > RateLimitAvoidanceThreshold)
{
// the number of call counts indicates we should throttle things a bit
if (InRateLimitAvoidanceModeStatus == false)
{
Logging.Log(Logging.LogType.Information, "API Connection", "Entered rate limit avoidance period, API calls will be throttled by " + RateLimitAvoidanceWait + " milliseconds.");
InRateLimitAvoidanceModeStatus = true;
}
return true;
}
else
{
// still in full speed mode - no throttle required
if (InRateLimitAvoidanceModeStatus == true)
{
Logging.Log(Logging.LogType.Information, "API Connection", "Exited rate limit avoidance period, API call rate is returned to full speed.");
InRateLimitAvoidanceModeStatus = false;
}
return false;
}
}
}
}
// rate limit handling - how long to wait to allow the server to recover and try again
// these values affect ALL communications if a 429 response code is received
///
/// How long to wait (in milliseconds) if a 429 status code is received before trying again
///
private static int RateLimitRecoveryWaitTime = 10000;
///
/// The time when normal communications can attempt to be resumed
///
private static DateTime RateLimitResumeTime = DateTime.UtcNow.AddMinutes(5 * -1);
// rate limit retry - how many times to retry before aborting
private int RetryAttempts = 0;
private int RetryAttemptsMax = 3;
///
/// Request data from the metadata API
///
/// Type of object to return
/// API endpoint segment to use
/// Fields to request from the API
/// Selection criteria for data to request
///
public async Task APIComm(string Endpoint, string Fields, string Query)
{
switch (_MetadataSource)
{
case HasheousClient.Models.MetadataModel.MetadataSources.None:
return null;
case HasheousClient.Models.MetadataModel.MetadataSources.IGDB:
return await IGDBAPI(Endpoint, Fields, Query);
default:
return null;
}
}
private async Task IGDBAPI(string Endpoint, string Fields, string Query)
{
Logging.Log(Logging.LogType.Debug, "API Connection", "Accessing API for endpoint: " + Endpoint);
if (RateLimitResumeTime > DateTime.UtcNow)
{
Logging.Log(Logging.LogType.Information, "API Connection", "IGDB rate limit hit. Pausing API communications until " + RateLimitResumeTime.ToString() + ". Attempt " + RetryAttempts + " of " + RetryAttemptsMax + " retries.");
Thread.Sleep(RateLimitRecoveryWaitTime);
}
try
{
if (InRateLimitAvoidanceMode == true)
{
// sleep for a moment to help avoid hitting the rate limiter
Thread.Sleep(RateLimitAvoidanceWait);
}
// perform the actual API call
var results = await igdb.QueryAsync(Endpoint, query: Fields + " " + Query + ";");
// increment rate limiter avoidance call count
RateLimitAvoidanceCallCount += 1;
return results;
}
catch (ApiException apiEx)
{
switch (apiEx.StatusCode)
{
case HttpStatusCode.TooManyRequests:
if (RetryAttempts >= RetryAttemptsMax)
{
Logging.Log(Logging.LogType.Warning, "API Connection", "IGDB rate limiter attempts expired. Aborting.", apiEx);
throw;
}
else
{
Logging.Log(Logging.LogType.Information, "API Connection", "IGDB API rate limit hit while accessing endpoint " + Endpoint, apiEx);
RetryAttempts += 1;
return await IGDBAPI(Endpoint, Fields, Query);
}
default:
Logging.Log(Logging.LogType.Warning, "API Connection", "Exception when accessing endpoint " + Endpoint, apiEx);
throw;
}
}
catch(Exception ex)
{
Logging.Log(Logging.LogType.Warning, "API Connection", "Exception when accessing endpoint " + Endpoint, ex);
throw;
}
}
///
/// Download from the specified uri
///
/// The uri to download from
/// The file name and path the download should be stored as
public Task DownloadFile(Uri uri, string DestinationFile)
{
var result = _DownloadFile(uri, DestinationFile);
return result;
}
private async Task _DownloadFile(Uri uri, string DestinationFile)
{
string DestinationDirectory = new FileInfo(DestinationFile).Directory.FullName;
if (!Directory.Exists(DestinationDirectory))
{
Directory.CreateDirectory(DestinationDirectory);
}
Logging.Log(Logging.LogType.Information, "Communications", "Downloading from " + uri.ToString() + " to " + DestinationFile);
try
{
using (HttpResponseMessage response = client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead).Result)
{
response.EnsureSuccessStatusCode();
using (Stream contentStream = await response.Content.ReadAsStreamAsync(), fileStream = new FileStream(DestinationFile, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true))
{
var totalRead = 0L;
var totalReads = 0L;
var buffer = new byte[8192];
var isMoreToRead = true;
do
{
var read = await contentStream.ReadAsync(buffer, 0, buffer.Length);
if (read == 0)
{
isMoreToRead = false;
}
else
{
await fileStream.WriteAsync(buffer, 0, read);
totalRead += read;
totalReads += 1;
if (totalReads % 2000 == 0)
{
Console.WriteLine(string.Format("total bytes downloaded so far: {0:n0}", totalRead));
}
}
}
while (isMoreToRead);
}
}
return true;
}
catch (HttpRequestException ex)
{
if (ex.StatusCode == HttpStatusCode.NotFound)
{
if (File.Exists(DestinationFile))
{
FileInfo fi = new FileInfo(DestinationFile);
if (fi.Length == 0)
{
File.Delete(DestinationFile);
}
}
}
Logging.Log(Logging.LogType.Warning, "Download Images", "Error downloading file: ", ex);
}
return false;
}
public async Task GetSpecificImageFromServer(string ImagePath, string ImageId, IGDBAPI_ImageSize size, List? FallbackSizes = null)
{
string returnPath = "";
// check for artificial sizes first
switch (size)
{
case IGDBAPI_ImageSize.screenshot_small:
case IGDBAPI_ImageSize.screenshot_thumb:
string BasePath = Path.Combine(ImagePath, size.ToString());
if (!Directory.Exists(BasePath))
{
Directory.CreateDirectory(BasePath);
}
returnPath = Path.Combine(BasePath, ImageId + ".jpg");
if (!File.Exists(returnPath))
{
// get original size image and resize
string originalSizePath = await GetSpecificImageFromServer(ImagePath, ImageId, IGDBAPI_ImageSize.original, null);
int width = 0;
int height = 0;
switch (size)
{
case IGDBAPI_ImageSize.screenshot_small:
// 235x128
width = 235;
height = 128;
break;
case IGDBAPI_ImageSize.screenshot_thumb:
// 165x90
width = 165;
height = 90;
break;
}
using (var image = new ImageMagick.MagickImage(originalSizePath))
{
image.Resize(width, height);
image.Strip();
image.Write(returnPath);
}
}
break;
default:
// these sizes are IGDB native
if (RateLimitResumeTime > DateTime.UtcNow)
{
Logging.Log(Logging.LogType.Information, "API Connection", "IGDB rate limit hit. Pausing API communications until " + RateLimitResumeTime.ToString() + ". Attempt " + RetryAttempts + " of " + RetryAttemptsMax + " retries.");
Thread.Sleep(RateLimitRecoveryWaitTime);
}
if (InRateLimitAvoidanceMode == true)
{
// sleep for a moment to help avoid hitting the rate limiter
Thread.Sleep(RateLimitAvoidanceWait);
}
Communications comms = new Communications();
List imageSizes = new List
{
size
};
// get the image
try
{
returnPath = Path.Combine(ImagePath, size.ToString(), ImageId + ".jpg");
// fail early if the file is already downloaded
if (!File.Exists(returnPath))
{
await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath);
}
}
catch (HttpRequestException ex)
{
if (ex.StatusCode == HttpStatusCode.NotFound)
{
Logging.Log(Logging.LogType.Information, "Image Download", "Image not found, trying a different size.");
if (FallbackSizes != null)
{
foreach (Communications.IGDBAPI_ImageSize imageSize in FallbackSizes)
{
returnPath = await GetSpecificImageFromServer(ImagePath, ImageId, imageSize, null);
}
}
}
}
// increment rate limiter avoidance call count
RateLimitAvoidanceCallCount += 1;
break;
}
return returnPath;
}
public static T? GetSearchCache(string SearchFields, string SearchString)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT * FROM SearchCache WHERE SearchFields = @searchfields AND SearchString = @searchstring;";
Dictionary dbDict = new Dictionary
{
{ "searchfields", SearchFields },
{ "searchstring", SearchString }
};
DataTable data = db.ExecuteCMD(sql, dbDict);
if (data.Rows.Count > 0)
{
// cache hit
string rawString = data.Rows[0]["Content"].ToString();
T ReturnValue = Newtonsoft.Json.JsonConvert.DeserializeObject(rawString);
if (ReturnValue != null)
{
Logging.Log(Logging.LogType.Information, "Search Cache", "Found search result in cache. Search string: " + SearchString);
return ReturnValue;
}
else
{
Logging.Log(Logging.LogType.Information, "Search Cache", "Search result not found in cache.");
return default;
}
}
else
{
// cache miss
Logging.Log(Logging.LogType.Information, "Search Cache", "Search result not found in cache.");
return default;
}
}
public static void SetSearchCache(string SearchFields, string SearchString, T SearchResult)
{
Logging.Log(Logging.LogType.Information, "Search Cache", "Storing search results in cache. Search string: " + SearchString);
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "INSERT INTO SearchCache (SearchFields, SearchString, Content, LastSearch) VALUES (@searchfields, @searchstring, @content, @lastsearch);";
Dictionary dbDict = new Dictionary
{
{ "searchfields", SearchFields },
{ "searchstring", SearchString },
{ "content", Newtonsoft.Json.JsonConvert.SerializeObject(SearchResult) },
{ "lastsearch", DateTime.UtcNow }
};
db.ExecuteNonQuery(sql, dbDict);
}
///
/// See https://api-docs.igdb.com/?javascript#images for more information about the image url structure
///
///
/// The path to save the downloaded files to
public async Task IGDBAPI_GetImage(List ImageSizes, string ImageId, string OutputPath)
{
string urlTemplate = "https://images.igdb.com/igdb/image/upload/t_{size}/{hash}.jpg";
foreach (IGDBAPI_ImageSize ImageSize in ImageSizes)
{
string url = urlTemplate.Replace("{size}", Common.GetDescription(ImageSize)).Replace("{hash}", ImageId);
string newOutputPath = Path.Combine(OutputPath, Common.GetDescription(ImageSize));
string OutputFile = ImageId + ".jpg";
string fullPath = Path.Combine(newOutputPath, OutputFile);
await _DownloadFile(new Uri(url), fullPath);
}
}
public enum IGDBAPI_ImageSize
{
///
/// 90x128 Fit
///
[Description("cover_small")]
cover_small,
///
/// 264x374 Fit
///
[Description("cover_big")]
cover_big,
///
/// 165x90 Lfill, Centre gravity - resized by Gaseous and is not a real IGDB size
///
[Description("screenshot_thumb")]
screenshot_thumb,
///
/// 235x128 Lfill, Centre gravity - resized by Gaseous and is not a real IGDB size
///
[Description("screenshot_small")]
screenshot_small,
///
/// 589x320 Lfill, Centre gravity
///
[Description("screenshot_med")]
screenshot_med,
///
/// 889x500 Lfill, Centre gravity
///
[Description("screenshot_big")]
screenshot_big,
///
/// 1280x720 Lfill, Centre gravity
///
[Description("screenshot_huge")]
screenshot_huge,
///
/// 284x160 Fit
///
[Description("logo_med")]
logo_med,
///
/// 90x90 Thumb, Centre gravity
///
[Description("thumb")]
thumb,
///
/// 35x35 Thumb, Centre gravity
///
[Description("micro")]
micro,
///
/// 1280x720 Fit, Centre gravity
///
[Description("720p")]
r720p,
///
/// 1920x1080 Fit, Centre gravity
///
[Description("1080p")]
r1080p,
///
/// The originally uploaded image
///
[Description("original")]
original
}
}
}