More refactoring and bug fixes (#243)

This commit is contained in:
Michael Green
2023-12-31 00:21:40 +11:00
committed by GitHub
parent 311c7733fa
commit 7be1ec7080
10 changed files with 230 additions and 193 deletions

View File

@@ -329,7 +329,8 @@ namespace gaseous_server.Classes
DataTable RetTable = new DataTable(); DataTable RetTable = new DataTable();
Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database", null, true); Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database", null, true);
MySqlConnection conn = new MySqlConnection(DBConn); using (MySqlConnection conn = new MySqlConnection(DBConn))
{
conn.Open(); conn.Open();
MySqlCommand cmd = new MySqlCommand MySqlCommand cmd = new MySqlCommand
@@ -361,6 +362,7 @@ namespace gaseous_server.Classes
Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection", null, true); Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection", null, true);
conn.Close(); conn.Close();
}
return RetTable; return RetTable;
} }
@@ -370,7 +372,8 @@ namespace gaseous_server.Classes
int result = 0; int result = 0;
Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database", null, true); Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database", null, true);
MySqlConnection conn = new MySqlConnection(DBConn); using (MySqlConnection conn = new MySqlConnection(DBConn))
{
conn.Open(); conn.Open();
MySqlCommand cmd = new MySqlCommand MySqlCommand cmd = new MySqlCommand
@@ -402,13 +405,15 @@ namespace gaseous_server.Classes
Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection", null, true); Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection", null, true);
conn.Close(); conn.Close();
}
return result; return result;
} }
public void TransactionExecCMD(List<Dictionary<string, object>> Parameters, int Timeout) public void TransactionExecCMD(List<Dictionary<string, object>> Parameters, int Timeout)
{ {
var conn = new MySqlConnection(DBConn); using (MySqlConnection conn = new MySqlConnection(DBConn))
{
conn.Open(); conn.Open();
var command = conn.CreateCommand(); var command = conn.CreateCommand();
MySqlTransaction transaction; MySqlTransaction transaction;
@@ -425,6 +430,7 @@ namespace gaseous_server.Classes
transaction.Commit(); transaction.Commit();
conn.Close(); conn.Close();
} }
}
private MySqlCommand buildcommand(MySqlConnection Conn, string SQL, Dictionary<string, object> Parameters, int Timeout) private MySqlCommand buildcommand(MySqlConnection Conn, string SQL, Dictionary<string, object> Parameters, int Timeout)
{ {
@@ -449,7 +455,8 @@ namespace gaseous_server.Classes
public bool TestConnection() public bool TestConnection()
{ {
MySqlConnection conn = new MySqlConnection(DBConn); using (MySqlConnection conn = new MySqlConnection(DBConn))
{
try try
{ {
conn.Open(); conn.Open();
@@ -463,5 +470,6 @@ namespace gaseous_server.Classes
} }
} }
} }
}
} }

View File

@@ -95,7 +95,8 @@ namespace gaseous_server.Classes.Metadata
{ {
Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Artwork download forced."); Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Artwork download forced.");
GetImageFromServer(ImagePath, returnValue.ImageId); Communications comms = new Communications();
comms.GetSpecificImageFromServer(ImagePath, returnValue.ImageId, Communications.IGDBAPI_ImageSize.original, null);
} }
return returnValue; return returnValue;
@@ -116,15 +117,6 @@ namespace gaseous_server.Classes.Metadata
return result; return result;
} }
private static async void GetImageFromServer(string ImagePath, string ImageId)
{
Communications comms = new Communications();
List<Communications.IGDBAPI_ImageSize> imageSizes = new List<Communications.IGDBAPI_ImageSize>();
imageSizes.AddRange(Enum.GetValues(typeof(Communications.IGDBAPI_ImageSize)).Cast<Communications.IGDBAPI_ImageSize>());
await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath);
}
} }
} }

View File

@@ -11,13 +11,23 @@ namespace gaseous_server.Classes.Metadata
/// </summary> /// </summary>
public class Communications public class Communications
{ {
static Communications()
{
var handler = new HttpClientHandler();
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
client = new HttpClient(handler);
client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip");
client.DefaultRequestHeaders.Add("Accept-Encoding", "deflate");
}
private static IGDBClient igdb = new IGDBClient( private static IGDBClient igdb = new IGDBClient(
// Found in Twitch Developer portal for your app // Found in Twitch Developer portal for your app
Config.IGDB.ClientId, Config.IGDB.ClientId,
Config.IGDB.Secret Config.IGDB.Secret
); );
private HttpClient client = new HttpClient(); private static HttpClient client = new HttpClient();
/// <summary> /// <summary>
/// Configure metadata API communications /// Configure metadata API communications
@@ -238,84 +248,101 @@ namespace gaseous_server.Classes.Metadata
private async Task<bool?> _DownloadFile(Uri uri, string DestinationFile) private async Task<bool?> _DownloadFile(Uri uri, string DestinationFile)
{ {
string DestinationDirectory = new FileInfo(DestinationFile).Directory.FullName; string DestinationDirectory = new FileInfo(DestinationFile).Directory.FullName;
Logging.Log(Logging.LogType.Information, "Communications", "Downloading from " + uri.ToString() + " to " + DestinationFile);
try
{
using (var s = client.GetStreamAsync(uri))
{
s.ConfigureAwait(false);
if (!Directory.Exists(DestinationDirectory)) if (!Directory.Exists(DestinationDirectory))
{ {
Directory.CreateDirectory(DestinationDirectory); Directory.CreateDirectory(DestinationDirectory);
} }
using (var fs = new FileStream(DestinationFile, FileMode.OpenOrCreate)) Logging.Log(Logging.LogType.Information, "Communications", "Downloading from " + uri.ToString() + " to " + DestinationFile);
try
{ {
s.Result.CopyTo(fs); using (HttpResponseMessage response = client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead).Result)
{
response.EnsureSuccessStatusCode();
using (Stream contentStream = await response.Content.ReadAsStreamAsync(), fileStream = new FileStream(DestinationFile, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true))
{
var totalRead = 0L;
var totalReads = 0L;
var buffer = new byte[8192];
var isMoreToRead = true;
do
{
var read = await contentStream.ReadAsync(buffer, 0, buffer.Length);
if (read == 0)
{
isMoreToRead = false;
}
else
{
await fileStream.WriteAsync(buffer, 0, read);
totalRead += read;
totalReads += 1;
if (totalReads % 2000 == 0)
{
Console.WriteLine(string.Format("total bytes downloaded so far: {0:n0}", totalRead));
}
}
}
while (isMoreToRead);
} }
} }
return true;
}
catch (HttpRequestException ex)
{
if (ex.StatusCode == HttpStatusCode.NotFound)
{
if (File.Exists(DestinationFile)) if (File.Exists(DestinationFile))
{ {
FileInfo fi = new FileInfo(DestinationFile); FileInfo fi = new FileInfo(DestinationFile);
if (fi.Length > 0) if (fi.Length == 0)
{
return true;
}
else
{ {
File.Delete(DestinationFile); File.Delete(DestinationFile);
throw new Exception("Zero length file");
} }
} }
else
{
throw new Exception("Zero length file");
} }
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Critical, "Communications", "Error while downloading from " + uri.ToString(), ex);
throw; Logging.Log(Logging.LogType.Warning, "Download Images", "Error downloading file: ", ex);
} }
return false; return false;
} }
public static async Task<string> GetSpecificImageFromServer(string ImagePath, string ImageId, IGDBAPI_ImageSize size, List<IGDBAPI_ImageSize>? FallbackSizes = null) public async Task<string> GetSpecificImageFromServer(string ImagePath, string ImageId, IGDBAPI_ImageSize size, List<IGDBAPI_ImageSize>? FallbackSizes = null)
{ {
string returnPath = "";
if (RateLimitResumeTime > DateTime.UtcNow)
{
Logging.Log(Logging.LogType.Information, "API Connection", "IGDB rate limit hit. Pausing API communications until " + RateLimitResumeTime.ToString() + ". Attempt " + RetryAttempts + " of " + RetryAttemptsMax + " retries.");
Thread.Sleep(RateLimitRecoveryWaitTime);
}
if (InRateLimitAvoidanceMode == true)
{
// sleep for a moment to help avoid hitting the rate limiter
Thread.Sleep(RateLimitAvoidanceWait);
}
Communications comms = new Communications(); Communications comms = new Communications();
List<IGDBAPI_ImageSize> imageSizes = new List<IGDBAPI_ImageSize> List<IGDBAPI_ImageSize> imageSizes = new List<IGDBAPI_ImageSize>
{ {
size size
}; };
string returnPath = "";
FileInfo? fi;
// get the image // get the image
try try
{ {
returnPath = Path.Combine(ImagePath, size.ToString(), ImageId + ".jpg"); returnPath = Path.Combine(ImagePath, size.ToString(), ImageId + ".jpg");
if (!File.Exists(returnPath)) if (File.Exists(returnPath)) { File.Delete(returnPath); }
{
await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath); await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath);
}
if (File.Exists(returnPath))
{
fi = new FileInfo(returnPath);
if (fi.Length == 0 || fi.LastWriteTimeUtc.AddDays(7) < DateTime.UtcNow)
{
File.Delete(returnPath);
await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath);
}
}
} }
catch (HttpRequestException ex) catch (HttpRequestException ex)
@@ -334,6 +361,9 @@ namespace gaseous_server.Classes.Metadata
} }
} }
// increment rate limiter avoidance call count
RateLimitAvoidanceCallCount += 1;
return returnPath; return returnPath;
} }
@@ -351,8 +381,25 @@ namespace gaseous_server.Classes.Metadata
string url = urlTemplate.Replace("{size}", Common.GetDescription(ImageSize)).Replace("{hash}", ImageId); string url = urlTemplate.Replace("{size}", Common.GetDescription(ImageSize)).Replace("{hash}", ImageId);
string newOutputPath = Path.Combine(OutputPath, Common.GetDescription(ImageSize)); string newOutputPath = Path.Combine(OutputPath, Common.GetDescription(ImageSize));
string OutputFile = ImageId + ".jpg"; string OutputFile = ImageId + ".jpg";
string fullPath = Path.Combine(newOutputPath, OutputFile);
await _DownloadFile(new Uri(url), Path.Combine(newOutputPath, OutputFile)); bool AllowDownload = true;
FileInfo fi;
if (File.Exists(fullPath))
{
fi = new FileInfo(fullPath);
if (fi.LastWriteTimeUtc.AddDays(14) < DateTime.UtcNow)
{
File.Delete(fullPath);
AllowDownload = true;
}
}
if (AllowDownload == true)
{
await _DownloadFile(new Uri(url), fullPath);
}
} }
} }

View File

@@ -97,7 +97,8 @@ namespace gaseous_server.Classes.Metadata
{ {
Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Company logo download forced."); Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Company logo download forced.");
GetImageFromServer(ImagePath, returnValue.ImageId); Communications comms = new Communications();
comms.GetSpecificImageFromServer(ImagePath, returnValue.ImageId, Communications.IGDBAPI_ImageSize.original, null);
} }
return returnValue; return returnValue;
@@ -118,15 +119,6 @@ namespace gaseous_server.Classes.Metadata
return result; return result;
} }
private static async void GetImageFromServer(string ImagePath, string ImageId)
{
Communications comms = new Communications();
List<Communications.IGDBAPI_ImageSize> imageSizes = new List<Communications.IGDBAPI_ImageSize>();
imageSizes.AddRange(Enum.GetValues(typeof(Communications.IGDBAPI_ImageSize)).Cast<Communications.IGDBAPI_ImageSize>());
await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath);
}
} }
} }

View File

@@ -96,14 +96,19 @@ namespace gaseous_server.Classes.Metadata
Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Cover download forced."); Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Cover download forced.");
// check for presence of image file - download if absent or force download is true // check for presence of image file - download if absent or force download is true
List<Communications.IGDBAPI_ImageSize> imageSizes = new List<Communications.IGDBAPI_ImageSize>(); List<Communications.IGDBAPI_ImageSize> imageSizes = new List<Communications.IGDBAPI_ImageSize>{
imageSizes.AddRange(Enum.GetValues(typeof(Communications.IGDBAPI_ImageSize)).Cast<Communications.IGDBAPI_ImageSize>()); Communications.IGDBAPI_ImageSize.cover_big,
Communications.IGDBAPI_ImageSize.cover_small,
Communications.IGDBAPI_ImageSize.original
};
Communications comms = new Communications();
foreach (Communications.IGDBAPI_ImageSize size in imageSizes) foreach (Communications.IGDBAPI_ImageSize size in imageSizes)
{ {
string localFile = Path.Combine(ImagePath, size.ToString(), returnValue.ImageId + ".jpg"); string localFile = Path.Combine(ImagePath, size.ToString(), returnValue.ImageId + ".jpg");
if ((!File.Exists(localFile)) || forceImageDownload == true) if ((!File.Exists(localFile)) || forceImageDownload == true)
{ {
Communications.GetSpecificImageFromServer(ImagePath, returnValue.ImageId, size, null); comms.GetSpecificImageFromServer(ImagePath, returnValue.ImageId, size, null);
} }
} }
} }
@@ -126,15 +131,6 @@ namespace gaseous_server.Classes.Metadata
return result; return result;
} }
public static async void GetImageFromServer(string ImagePath, string ImageId)
{
Communications comms = new Communications();
List<Communications.IGDBAPI_ImageSize> imageSizes = new List<Communications.IGDBAPI_ImageSize>();
imageSizes.AddRange(Enum.GetValues(typeof(Communications.IGDBAPI_ImageSize)).Cast<Communications.IGDBAPI_ImageSize>());
await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath);
}
} }
} }

View File

@@ -99,7 +99,8 @@ namespace gaseous_server.Classes.Metadata
{ {
Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Platform logo download forced."); Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Platform logo download forced.");
GetImageFromServer(ImagePath, returnValue.ImageId); Communications comms = new Communications();
comms.GetSpecificImageFromServer(ImagePath, returnValue.ImageId, Communications.IGDBAPI_ImageSize.original, null);
} }
} }
@@ -121,15 +122,6 @@ namespace gaseous_server.Classes.Metadata
return result; return result;
} }
private static async void GetImageFromServer(string ImagePath, string ImageId)
{
Communications comms = new Communications();
List<Communications.IGDBAPI_ImageSize> imageSizes = new List<Communications.IGDBAPI_ImageSize>();
imageSizes.AddRange(Enum.GetValues(typeof(Communications.IGDBAPI_ImageSize)).Cast<Communications.IGDBAPI_ImageSize>());
comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath);
}
} }
} }

View File

@@ -95,7 +95,8 @@ namespace gaseous_server.Classes.Metadata
{ {
Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Screenshot download forced."); Logging.Log(Logging.LogType.Information, "Metadata: " + returnValue.GetType().Name, "Screenshot download forced.");
GetImageFromServer(ImagePath, returnValue.ImageId); Communications comms = new Communications();
comms.GetSpecificImageFromServer(ImagePath, returnValue.ImageId, Communications.IGDBAPI_ImageSize.original, null);
} }
return returnValue; return returnValue;
@@ -116,16 +117,6 @@ namespace gaseous_server.Classes.Metadata
return result; return result;
} }
private static async void GetImageFromServer(string ImagePath, string ImageId)
{
Communications comms = new Communications();
List<Communications.IGDBAPI_ImageSize> imageSizes = new List<Communications.IGDBAPI_ImageSize>();
imageSizes.Add(Communications.IGDBAPI_ImageSize.original);
imageSizes.Add(Communications.IGDBAPI_ImageSize.thumb);
await comms.IGDBAPI_GetImage(imageSizes, ImageId, ImagePath);
}
} }
} }

View File

@@ -13,7 +13,7 @@ namespace gaseous_server.Classes
DataTable dt = new DataTable(); DataTable dt = new DataTable();
// disabling forceRefresh // disabling forceRefresh
forceRefresh = true; forceRefresh = false;
// update platforms // update platforms
sql = "SELECT Id, `Name` FROM Platform;"; sql = "SELECT Id, `Name` FROM Platform;";

View File

@@ -537,8 +537,18 @@ namespace gaseous_server.Controllers
try try
{ {
IGDB.Models.Artwork artworkObject = Artworks.GetArtwork(ArtworkId, Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject)); IGDB.Models.Artwork artworkObject = Artworks.GetArtwork(ArtworkId, Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject));
if (artworkObject != null) { if (artworkObject != null) {
string coverFilePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), "Artwork", size.ToString(), artworkObject.ImageId + ".jpg"); //string coverFilePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), "Artwork", size.ToString(), artworkObject.ImageId + ".jpg");
string basePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), "Artwork");
Communications comms = new Communications();
Task<string> ImgFetch = comms.GetSpecificImageFromServer(basePath, artworkObject.ImageId, size, new List<Communications.IGDBAPI_ImageSize>{ Communications.IGDBAPI_ImageSize.original });
string coverFilePath = ImgFetch.Result;
if (System.IO.File.Exists(coverFilePath)) if (System.IO.File.Exists(coverFilePath))
{ {
string filename = artworkObject.ImageId + ".jpg"; string filename = artworkObject.ImageId + ".jpg";
@@ -633,7 +643,8 @@ namespace gaseous_server.Controllers
IGDB.Models.Cover cover = Classes.Metadata.Covers.GetCover(gameObject.Cover.Id, Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), false); IGDB.Models.Cover cover = Classes.Metadata.Covers.GetCover(gameObject.Cover.Id, Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), false);
string basePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), "Covers"); string basePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), "Covers");
Task<string> ImgFetch = Communications.GetSpecificImageFromServer(basePath, cover.ImageId, size, new List<Communications.IGDBAPI_ImageSize>{ Communications.IGDBAPI_ImageSize.cover_big, Communications.IGDBAPI_ImageSize.original }); Communications comms = new Communications();
Task<string> ImgFetch = comms.GetSpecificImageFromServer(basePath, cover.ImageId, size, new List<Communications.IGDBAPI_ImageSize>{ Communications.IGDBAPI_ImageSize.cover_big, Communications.IGDBAPI_ImageSize.original });
string coverFilePath = ImgFetch.Result; string coverFilePath = ImgFetch.Result;
@@ -1346,7 +1357,15 @@ namespace gaseous_server.Controllers
IGDB.Models.Screenshot screenshotObject = Screenshots.GetScreenshot(ScreenshotId, Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject)); IGDB.Models.Screenshot screenshotObject = Screenshots.GetScreenshot(ScreenshotId, Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject));
string coverFilePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), "Screenshots", Size.ToString(), screenshotObject.ImageId + ".jpg"); //string coverFilePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), "Screenshots", Size.ToString(), screenshotObject.ImageId + ".jpg");
string basePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(gameObject), "Screenshots");
Communications comms = new Communications();
Task<string> ImgFetch = comms.GetSpecificImageFromServer(basePath, screenshotObject.ImageId, Size, new List<Communications.IGDBAPI_ImageSize>{ Communications.IGDBAPI_ImageSize.original });
string coverFilePath = ImgFetch.Result;
if (System.IO.File.Exists(coverFilePath)) if (System.IO.File.Exists(coverFilePath))
{ {
string filename = screenshotObject.ImageId + ".jpg"; string filename = screenshotObject.ImageId + ".jpg";

View File

@@ -27,12 +27,12 @@
} else { } else {
if (result.cover) { if (result.cover) {
var bg = document.getElementById('bgImage'); var bg = document.getElementById('bgImage');
bg.setAttribute('style', 'background-image: url("/api/v1.1/Games/' + gameId + '/cover/image"); background-position: center; background-repeat: no-repeat; background-size: cover; filter: blur(10px); -webkit-filter: blur(10px);'); bg.setAttribute('style', 'background-image: url("/api/v1.1/Games/' + gameId + '/cover/image/original"); background-position: center; background-repeat: no-repeat; background-size: cover; filter: blur(10px); -webkit-filter: blur(10px);');
} }
} }
if (result.cover) { if (result.cover) {
emuBackground = '/api/v1.1/Games/' + gameId + '/cover/image'; emuBackground = '/api/v1.1/Games/' + gameId + '/cover/image/original';
} }
emuGameTitle = gameData.name; emuGameTitle = gameData.name;
@@ -59,7 +59,7 @@
artworksPosition = 0; artworksPosition = 0;
} }
var bg = document.getElementById('bgImage'); var bg = document.getElementById('bgImage');
bg.setAttribute('style', 'background-image: url("/api/v1.1/Games/' + gameId + '/artwork/' + artworks[artworksPosition] + '/image"); background-position: center; background-repeat: no-repeat; background-size: cover; filter: blur(10px); -webkit-filter: blur(10px);'); bg.setAttribute('style', 'background-image: url("/api/v1.1/Games/' + gameId + '/artwork/' + artworks[artworksPosition] + '/image/original"); background-position: center; background-repeat: no-repeat; background-size: cover; filter: blur(10px); -webkit-filter: blur(10px);');
} }
} }
</script> </script>