using System.ComponentModel; using System.Data; using System.Drawing; using System.Net; using Humanizer; using IGDB; using Microsoft.CodeAnalysis.CSharp.Syntax; using RestEase; namespace gaseous_server.Classes.Metadata { /// /// Handles all metadata API communications /// public class Communications { static Communications() { var handler = new HttpClientHandler(); handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; client = new HttpClient(handler); client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip"); client.DefaultRequestHeaders.Add("Accept-Encoding", "deflate"); } private static IGDBClient igdb = new IGDBClient( // Found in Twitch Developer portal for your app Config.IGDB.ClientId, Config.IGDB.Secret ); private static HttpClient client = new HttpClient(); /// /// Configure metadata API communications /// public static HasheousClient.Models.MetadataModel.MetadataSources MetadataSource { get { return _MetadataSource; } set { _MetadataSource = value; switch (value) { case HasheousClient.Models.MetadataModel.MetadataSources.IGDB: // set rate limiter avoidance values RateLimitAvoidanceWait = 1500; RateLimitAvoidanceThreshold = 3; RateLimitAvoidancePeriod = 1; // set rate limiter recovery values RateLimitRecoveryWaitTime = 10000; break; default: // leave all values at default break; } } } private static HasheousClient.Models.MetadataModel.MetadataSources _MetadataSource = HasheousClient.Models.MetadataModel.MetadataSources.None; // rate limit avoidance - what can we do to ensure that rate limiting is avoided? // these values affect all communications /// /// How long to wait to avoid hitting an API rate limiter /// private static int RateLimitAvoidanceWait = 2000; /// /// How many API calls in the period are allowed before we start introducing a wait /// private static int RateLimitAvoidanceThreshold = 80; /// /// A counter of API calls since the beginning of the period /// private static int RateLimitAvoidanceCallCount = 0; /// /// How large the period (in seconds) to measure API call counts against /// private static int RateLimitAvoidancePeriod = 60; /// /// The start of the rate limit avoidance period /// private static DateTime RateLimitAvoidanceStartTime = DateTime.UtcNow; /// /// Used to determine if we're already in rate limit avoidance mode - always query "InRateLimitAvoidanceMode" /// for up to date mode status. /// This bool is used to track status changes and should not be relied upon for current status. /// private static bool InRateLimitAvoidanceModeStatus = false; /// /// Determine if we're in rate limit avoidance mode. /// private static bool InRateLimitAvoidanceMode { get { if (RateLimitAvoidanceStartTime.AddSeconds(RateLimitAvoidancePeriod) <= DateTime.UtcNow) { // avoidance period has expired - reset RateLimitAvoidanceCallCount = 0; RateLimitAvoidanceStartTime = DateTime.UtcNow; return false; } else { // we're in the avoidance period if (RateLimitAvoidanceCallCount > RateLimitAvoidanceThreshold) { // the number of call counts indicates we should throttle things a bit if (InRateLimitAvoidanceModeStatus == false) { Logging.Log(Logging.LogType.Information, "API Connection", "Entered rate limit avoidance period, API calls will be throttled by " + RateLimitAvoidanceWait + " milliseconds."); InRateLimitAvoidanceModeStatus = true; } return true; } else { // still in full speed mode - no throttle required if (InRateLimitAvoidanceModeStatus == true) { Logging.Log(Logging.LogType.Information, "API Connection", "Exited rate limit avoidance period, API call rate is returned to full speed."); InRateLimitAvoidanceModeStatus = false; } return false; } } } } // rate limit handling - how long to wait to allow the server to recover and try again // these values affect ALL communications if a 429 response code is received /// /// How long to wait (in milliseconds) if a 429 status code is received before trying again /// private static int RateLimitRecoveryWaitTime = 10000; /// /// The time when normal communications can attempt to be resumed /// private static DateTime RateLimitResumeTime = DateTime.UtcNow.AddMinutes(5 * -1); // rate limit retry - how many times to retry before aborting private int RetryAttempts = 0; private int RetryAttemptsMax = 3; /// /// Request data from the metadata API /// /// Type of object to return /// API endpoint segment to use /// Fields to request from the API /// Selection criteria for data to request /// public async Task APIComm(string Endpoint, string Fields, string Query) { switch (_MetadataSource) { case HasheousClient.Models.MetadataModel.MetadataSources.None: return null; case HasheousClient.Models.MetadataModel.MetadataSources.IGDB: return await IGDBAPI(Endpoint, Fields, Query); default: return null; } } private async Task IGDBAPI(string Endpoint, string Fields, string Query) { Logging.Log(Logging.LogType.Debug, "API Connection", "Accessing API for endpoint: " + Endpoint); if (RateLimitResumeTime > DateTime.UtcNow) { Logging.Log(Logging.LogType.Information, "API Connection", "IGDB rate limit hit. Pausing API communications until " + RateLimitResumeTime.ToString() + ". Attempt " + RetryAttempts + " of " + RetryAttemptsMax + " retries."); Thread.Sleep(RateLimitRecoveryWaitTime); } try { if (InRateLimitAvoidanceMode == true) { // sleep for a moment to help avoid hitting the rate limiter Thread.Sleep(RateLimitAvoidanceWait); } // perform the actual API call var results = await igdb.QueryAsync(Endpoint, query: Fields + " " + Query + ";"); // increment rate limiter avoidance call count RateLimitAvoidanceCallCount += 1; return results; } catch (ApiException apiEx) { switch (apiEx.StatusCode) { case HttpStatusCode.TooManyRequests: if (RetryAttempts >= RetryAttemptsMax) { Logging.Log(Logging.LogType.Warning, "API Connection", "IGDB rate limiter attempts expired. Aborting.", apiEx); throw; } else { Logging.Log(Logging.LogType.Information, "API Connection", "IGDB API rate limit hit while accessing endpoint " + Endpoint, apiEx); RetryAttempts += 1; return await IGDBAPI(Endpoint, Fields, Query); } default: Logging.Log(Logging.LogType.Warning, "API Connection", "Exception when accessing endpoint " + Endpoint, apiEx); throw; } } catch(Exception ex) { Logging.Log(Logging.LogType.Warning, "API Connection", "Exception when accessing endpoint " + Endpoint, ex); throw; } } /// /// Download from the specified uri /// /// The uri to download from /// The file name and path the download should be stored as public Task DownloadFile(Uri uri, string DestinationFile) { var result = _DownloadFile(uri, DestinationFile); return result; } private async Task _DownloadFile(Uri uri, string DestinationFile) { string DestinationDirectory = new FileInfo(DestinationFile).Directory.FullName; if (!Directory.Exists(DestinationDirectory)) { Directory.CreateDirectory(DestinationDirectory); } Logging.Log(Logging.LogType.Information, "Communications", "Downloading from " + uri.ToString() + " to " + DestinationFile); try { using (HttpResponseMessage response = client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead).Result) { response.EnsureSuccessStatusCode(); using (Stream contentStream = await response.Content.ReadAsStreamAsync(), fileStream = new FileStream(DestinationFile, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true)) { var totalRead = 0L; var totalReads = 0L; var buffer = new byte[8192]; var isMoreToRead = true; do { var read = await contentStream.ReadAsync(buffer, 0, buffer.Length); if (read == 0) { isMoreToRead = false; } else { await fileStream.WriteAsync(buffer, 0, read); totalRead += read; totalReads += 1; if (totalReads % 2000 == 0) { Console.WriteLine(string.Format("total bytes downloaded so far: {0:n0}", totalRead)); } } } while (isMoreToRead); } } return true; } catch (HttpRequestException ex) { if (ex.StatusCode == HttpStatusCode.NotFound) { if (File.Exists(DestinationFile)) { FileInfo fi = new FileInfo(DestinationFile); if (fi.Length == 0) { File.Delete(DestinationFile); } } } Logging.Log(Logging.LogType.Warning, "Download Images", "Error downloading file: ", ex); } return false; } public async Task GetSpecificImageFromServer(string ImagePath, string ImageId, IGDBAPI_ImageSize size, List? FallbackSizes = null) { string returnPath = ""; // check for artificial sizes first switch (size) { case IGDBAPI_ImageSize.screenshot_small: case IGDBAPI_ImageSize.screenshot_thumb: string BasePath = Path.Combine(ImagePath, size.ToString()); if (!Directory.Exists(BasePath)) { Directory.CreateDirectory(BasePath); } returnPath = Path.Combine(BasePath, ImageId + ".jpg"); if (!File.Exists(returnPath)) { // get original size image and resize string originalSizePath = await GetSpecificImageFromServer(ImagePath, ImageId, IGDBAPI_ImageSize.original, null); int width = 0; int height = 0; switch (size) { case IGDBAPI_ImageSize.screenshot_small: // 235x128 width = 235; height = 128; break; case IGDBAPI_ImageSize.screenshot_thumb: // 165x90 width = 165; height = 90; break; } using (var image = new ImageMagick.MagickImage(originalSizePath)) { image.Resize(width, height); image.Strip(); image.Write(returnPath); } } break; default: // these sizes are IGDB native if (RateLimitResumeTime > DateTime.UtcNow) { Logging.Log(Logging.LogType.Information, "API Connection", "IGDB rate limit hit. Pausing API communications until " + RateLimitResumeTime.ToString() + ". Attempt " + RetryAttempts + " of " + RetryAttemptsMax + " retries."); Thread.Sleep(RateLimitRecoveryWaitTime); } if (InRateLimitAvoidanceMode == true) { // sleep for a moment to help avoid hitting the rate limiter Thread.Sleep(RateLimitAvoidanceWait); } Communications comms = new Communications(); List imageSizes = new List { size }; // get the image try { returnPath = Path.Combine(ImagePath, size.ToString(), ImageId + ".jpg"); // fail early if the file is already downloaded if (!File.Exists(returnPath)) { await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath); } } catch (HttpRequestException ex) { if (ex.StatusCode == HttpStatusCode.NotFound) { Logging.Log(Logging.LogType.Information, "Image Download", "Image not found, trying a different size."); if (FallbackSizes != null) { foreach (Communications.IGDBAPI_ImageSize imageSize in FallbackSizes) { returnPath = await GetSpecificImageFromServer(ImagePath, ImageId, imageSize, null); } } } } // increment rate limiter avoidance call count RateLimitAvoidanceCallCount += 1; break; } return returnPath; } public static T? GetSearchCache(string SearchFields, string SearchString) { Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); string sql = "SELECT * FROM SearchCache WHERE SearchFields = @searchfields AND SearchString = @searchstring;"; Dictionary dbDict = new Dictionary { { "searchfields", SearchFields }, { "searchstring", SearchString } }; DataTable data = db.ExecuteCMD(sql, dbDict); if (data.Rows.Count > 0) { // cache hit string rawString = data.Rows[0]["Content"].ToString(); T ReturnValue = Newtonsoft.Json.JsonConvert.DeserializeObject(rawString); if (ReturnValue != null) { Logging.Log(Logging.LogType.Information, "Search Cache", "Found search result in cache. Search string: " + SearchString); return ReturnValue; } else { Logging.Log(Logging.LogType.Information, "Search Cache", "Search result not found in cache."); return default; } } else { // cache miss Logging.Log(Logging.LogType.Information, "Search Cache", "Search result not found in cache."); return default; } } public static void SetSearchCache(string SearchFields, string SearchString, T SearchResult) { Logging.Log(Logging.LogType.Information, "Search Cache", "Storing search results in cache. Search string: " + SearchString); Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); string sql = "INSERT INTO SearchCache (SearchFields, SearchString, Content, LastSearch) VALUES (@searchfields, @searchstring, @content, @lastsearch);"; Dictionary dbDict = new Dictionary { { "searchfields", SearchFields }, { "searchstring", SearchString }, { "content", Newtonsoft.Json.JsonConvert.SerializeObject(SearchResult) }, { "lastsearch", DateTime.UtcNow } }; db.ExecuteNonQuery(sql, dbDict); } /// /// See https://api-docs.igdb.com/?javascript#images for more information about the image url structure /// /// /// The path to save the downloaded files to public async Task IGDBAPI_GetImage(List ImageSizes, string ImageId, string OutputPath) { string urlTemplate = "https://images.igdb.com/igdb/image/upload/t_{size}/{hash}.jpg"; foreach (IGDBAPI_ImageSize ImageSize in ImageSizes) { string url = urlTemplate.Replace("{size}", Common.GetDescription(ImageSize)).Replace("{hash}", ImageId); string newOutputPath = Path.Combine(OutputPath, Common.GetDescription(ImageSize)); string OutputFile = ImageId + ".jpg"; string fullPath = Path.Combine(newOutputPath, OutputFile); await _DownloadFile(new Uri(url), fullPath); } } public enum IGDBAPI_ImageSize { /// /// 90x128 Fit /// [Description("cover_small")] cover_small, /// /// 264x374 Fit /// [Description("cover_big")] cover_big, /// /// 165x90 Lfill, Centre gravity - resized by Gaseous and is not a real IGDB size /// [Description("screenshot_thumb")] screenshot_thumb, /// /// 235x128 Lfill, Centre gravity - resized by Gaseous and is not a real IGDB size /// [Description("screenshot_small")] screenshot_small, /// /// 589x320 Lfill, Centre gravity /// [Description("screenshot_med")] screenshot_med, /// /// 889x500 Lfill, Centre gravity /// [Description("screenshot_big")] screenshot_big, /// /// 1280x720 Lfill, Centre gravity /// [Description("screenshot_huge")] screenshot_huge, /// /// 284x160 Fit /// [Description("logo_med")] logo_med, /// /// 90x90 Thumb, Centre gravity /// [Description("thumb")] thumb, /// /// 35x35 Thumb, Centre gravity /// [Description("micro")] micro, /// /// 1280x720 Fit, Centre gravity /// [Description("720p")] r720p, /// /// 1920x1080 Fit, Centre gravity /// [Description("1080p")] r1080p, /// /// The originally uploaded image /// [Description("original")] original } } }