From 789ec7fc17d07218c2ffba5c7e41008e1253e04e Mon Sep 17 00:00:00 2001 From: Michael Green <84688932+michael-j-green@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:42:40 +1100 Subject: [PATCH] Improve background task progress feedback (#228) * Include a last run duration field for background tasks * Improved background task progress feedback --- gaseous-server/Classes/ImportGames.cs | 26 +++++++++-- gaseous-server/Classes/Maintenance.cs | 12 +++-- gaseous-server/Classes/MetadataManagement.cs | 23 ++++++++-- gaseous-server/Classes/QueueItemStatus.cs | 41 +++++++++++++++++ .../Classes/SignatureIngestors/XML.cs | 5 ++- gaseous-server/ProcessQueue.cs | 45 +++++++++++++++---- .../wwwroot/pages/settings/system.html | 30 ++++++++----- 7 files changed, 151 insertions(+), 31 deletions(-) create mode 100644 gaseous-server/Classes/QueueItemStatus.cs diff --git a/gaseous-server/Classes/ImportGames.cs b/gaseous-server/Classes/ImportGames.cs index f139d91..3abdcde 100644 --- a/gaseous-server/Classes/ImportGames.cs +++ b/gaseous-server/Classes/ImportGames.cs @@ -1,17 +1,19 @@ using System; using System.Data; using System.IO.Compression; +using System.Security.Authentication; using System.Security.Policy; using System.Text.RegularExpressions; using System.Threading.Tasks; using gaseous_server.Classes.Metadata; using IGDB.Models; using NuGet.Common; +using NuGet.LibraryModel; using static gaseous_server.Classes.Metadata.Games; namespace gaseous_server.Classes { - public class ImportGames + public class ImportGames : QueueItemStatus { public ImportGames(string ImportPath) { @@ -21,9 +23,15 @@ namespace gaseous_server.Classes string[] importContents_Directories = Directory.GetDirectories(ImportPath); // import files first + int importCount = 1; foreach (string importContent in importContents_Files) { + SetStatus(importCount, importContents_Files.Length, "Importing file: " + importContent); + ImportGame.ImportGameFile(importContent, null); + + importCount += 1; } + ClearStatus(); // import sub directories foreach (string importDir in importContents_Directories) { @@ -40,7 +48,7 @@ namespace gaseous_server.Classes } - public class ImportGame + public class ImportGame : QueueItemStatus { public static void ImportGameFile(string GameFileImportPath, IGDB.Models.Platform? OverridePlatform) { @@ -600,7 +608,7 @@ namespace gaseous_server.Classes } } - public static void LibraryScan() + public void LibraryScan() { foreach (GameLibrary.LibraryItem library in GameLibrary.GetLibraries) { @@ -645,8 +653,10 @@ namespace gaseous_server.Classes // search for files in the library that aren't in the database Logging.Log(Logging.LogType.Information, "Library Scan", "Looking for orphaned library files to add"); string[] LibraryFiles = Directory.GetFiles(library.Path, "*.*", SearchOption.AllDirectories); + int StatusCount = 0; foreach (string LibraryFile in LibraryFiles) { + SetStatus(StatusCount, LibraryFiles.Length, "Processing file " + LibraryFile); if (!Common.SkippableFiles.Contains(Path.GetFileName(LibraryFile), StringComparer.OrdinalIgnoreCase)) { Common.hashObject LibraryFileHash = new Common.hashObject(LibraryFile); @@ -702,6 +712,7 @@ namespace gaseous_server.Classes } } } + ClearStatus(); sql = "SELECT * FROM Games_Roms WHERE LibraryId=@libraryid ORDER BY `name`"; dtRoms = db.ExecuteCMD(sql, dbDict); @@ -746,7 +757,7 @@ namespace gaseous_server.Classes } } - public static void Rematcher(bool ForceExecute = false) + public void Rematcher(bool ForceExecute = false) { // rescan all titles with an unknown platform or title and see if we can get a match Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan starting"); @@ -764,8 +775,11 @@ namespace gaseous_server.Classes Dictionary dbDict = new Dictionary(); dbDict.Add("lastmatchattemptdate", DateTime.UtcNow.AddDays(-7)); DataTable data = db.ExecuteCMD(sql, dbDict); + int StatusCount = -0; foreach (DataRow row in data.Rows) { + SetStatus(StatusCount, data.Rows.Count, "Running rematcher"); + // get library GameLibrary.LibraryItem library = GameLibrary.GetLibrary((int)row["LibraryId"]); @@ -800,9 +814,13 @@ namespace gaseous_server.Classes dbLastAttemptDict.Add("id", romId); dbLastAttemptDict.Add("lastmatchattemptdate", DateTime.UtcNow); db.ExecuteCMD(attemptSql, dbLastAttemptDict); + + StatusCount += 1; } + ClearStatus(); Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan completed"); + ClearStatus(); } } } diff --git a/gaseous-server/Classes/Maintenance.cs b/gaseous-server/Classes/Maintenance.cs index 14a8caf..662b96e 100644 --- a/gaseous-server/Classes/Maintenance.cs +++ b/gaseous-server/Classes/Maintenance.cs @@ -5,11 +5,11 @@ using Microsoft.VisualStudio.Web.CodeGeneration; namespace gaseous_server.Classes { - public class Maintenance + public class Maintenance : QueueItemStatus { const int MaxFileAge = 30; - public static void RunMaintenance() + public void RunMaintenance() { // delete files and directories older than 7 days in PathsToClean List PathsToClean = new List(); @@ -49,8 +49,11 @@ namespace gaseous_server.Classes string sql = "SHOW TABLES;"; DataTable tables = db.ExecuteCMD(sql); + int StatusCounter = 1; foreach (DataRow row in tables.Rows) { + SetStatus(StatusCounter, tables.Rows.Count, "Optimising table " + row[0].ToString()); + sql = "OPTIMIZE TABLE " + row[0].ToString(); DataTable response = db.ExecuteCMD(sql); foreach (DataRow responseRow in response.Rows) @@ -60,9 +63,12 @@ namespace gaseous_server.Classes { retVal += responseRow.ItemArray[i] + "; "; } - Logging.Log(Logging.LogType.Information, "Maintenance", "Optimise table " + row[0].ToString() + ": " + retVal); + Logging.Log(Logging.LogType.Information, "Maintenance", "(" + StatusCounter + "/" + tables.Rows.Count + "): Optimise table " + row[0].ToString() + ": " + retVal); } + + StatusCounter += 1; } + ClearStatus(); } } } \ No newline at end of file diff --git a/gaseous-server/Classes/MetadataManagement.cs b/gaseous-server/Classes/MetadataManagement.cs index 9db4e2c..09ebe52 100644 --- a/gaseous-server/Classes/MetadataManagement.cs +++ b/gaseous-server/Classes/MetadataManagement.cs @@ -4,30 +4,39 @@ using gaseous_server.Models; namespace gaseous_server.Classes { - public class MetadataManagement + public class MetadataManagement : QueueItemStatus { - public static void RefreshMetadata(bool forceRefresh = false) + public void RefreshMetadata(bool forceRefresh = false) { Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); string sql = ""; DataTable dt = new DataTable(); + // disabling forceRefresh + forceRefresh = false; + // update platforms sql = "SELECT Id, `Name` FROM Platform;"; dt = db.ExecuteCMD(sql); + int StatusCounter = 1; foreach (DataRow dr in dt.Rows) { + SetStatus(StatusCounter, dt.Rows.Count, "Refreshing metadata for platform " + dr["name"]); + try { - Logging.Log(Logging.LogType.Information, "Metadata Refresh", "Refreshing metadata for platform " + dr["name"] + " (" + dr["id"] + ")"); + Logging.Log(Logging.LogType.Information, "Metadata Refresh", "(" + StatusCounter + "/" + dt.Rows.Count + "): Refreshing metadata for platform " + dr["name"] + " (" + dr["id"] + ")"); Metadata.Platforms.GetPlatform((long)dr["id"], true); } catch (Exception ex) { Logging.Log(Logging.LogType.Critical, "Metadata Refresh", "An error occurred while refreshing metadata for " + dr["name"], ex); } + + StatusCounter += 1; } + ClearStatus(); // update games if (forceRefresh == true) @@ -42,18 +51,24 @@ namespace gaseous_server.Classes } dt = db.ExecuteCMD(sql); + StatusCounter = 1; foreach (DataRow dr in dt.Rows) { + SetStatus(StatusCounter, dt.Rows.Count, "Refreshing metadata for game " + dr["name"]); + try { - Logging.Log(Logging.LogType.Information, "Metadata Refresh", "Refreshing metadata for game " + dr["name"] + " (" + dr["id"] + ")"); + Logging.Log(Logging.LogType.Information, "Metadata Refresh", "(" + StatusCounter + "/" + dt.Rows.Count + "): Refreshing metadata for game " + dr["name"] + " (" + dr["id"] + ")"); Metadata.Games.GetGame((long)dr["id"], true, false, forceRefresh); } catch (Exception ex) { Logging.Log(Logging.LogType.Critical, "Metadata Refresh", "An error occurred while refreshing metadata for " + dr["name"], ex); } + + StatusCounter += 1; } + ClearStatus(); } } } diff --git a/gaseous-server/Classes/QueueItemStatus.cs b/gaseous-server/Classes/QueueItemStatus.cs new file mode 100644 index 0000000..51bd913 --- /dev/null +++ b/gaseous-server/Classes/QueueItemStatus.cs @@ -0,0 +1,41 @@ +namespace gaseous_server.Classes +{ + public class QueueItemStatus + { + internal ProcessQueue.QueueItem? CallingQueueItem = null; + + private int _CurrentItemNumber = 0; + private int _MaxItemsNumber = 0; + private string _StatusText = ""; + + public int CurrentItemNumber => _CurrentItemNumber; + public int MaxItemsNumber => _MaxItemsNumber; + public string StatusText => _StatusText; + + public void SetStatus(int CurrentItemNumber, int MaxItemsNumber, string StatusText) + { + this._CurrentItemNumber = CurrentItemNumber; + this._MaxItemsNumber = MaxItemsNumber; + this._StatusText = StatusText; + + if (CallingQueueItem != null) + { + CallingQueueItem.CurrentState = _CurrentItemNumber + " of " + _MaxItemsNumber + ": " + _StatusText; + CallingQueueItem.CurrentStateProgress = _CurrentItemNumber + " of " + _MaxItemsNumber; + } + } + + public void ClearStatus() + { + this._CurrentItemNumber = 0; + this._MaxItemsNumber = 0; + this._StatusText = ""; + + if (CallingQueueItem != null) + { + CallingQueueItem.CurrentState = ""; + CallingQueueItem.CurrentStateProgress = ""; + } + } + } +} \ No newline at end of file diff --git a/gaseous-server/Classes/SignatureIngestors/XML.cs b/gaseous-server/Classes/SignatureIngestors/XML.cs index b1c2473..8378df0 100644 --- a/gaseous-server/Classes/SignatureIngestors/XML.cs +++ b/gaseous-server/Classes/SignatureIngestors/XML.cs @@ -6,7 +6,7 @@ using System.Data; namespace gaseous_server.SignatureIngestors.XML { - public class XMLIngestor + public class XMLIngestor : QueueItemStatus { public void Import(string SearchPath, gaseous_signature_parser.parser.SignatureParser XMLType) { @@ -31,6 +31,8 @@ namespace gaseous_server.SignatureIngestors.XML { string XMLFile = PathContents[i]; + SetStatus(i + 1, PathContents.Length, "Processing signature file: " + XMLFile); + // check xml file md5 Common.hashObject hashObject = new Common.hashObject(XMLFile); sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; @@ -247,6 +249,7 @@ namespace gaseous_server.SignatureIngestors.XML Logging.Log(Logging.LogType.Debug, "Signature Ingestor - XML", "Rejecting already imported file: " + XMLFile); } } + ClearStatus(); } } } \ No newline at end of file diff --git a/gaseous-server/ProcessQueue.cs b/gaseous-server/ProcessQueue.cs index e316a89..d1f4f41 100644 --- a/gaseous-server/ProcessQueue.cs +++ b/gaseous-server/ProcessQueue.cs @@ -33,6 +33,7 @@ namespace gaseous_server private QueueItemType _ItemType = QueueItemType.NotConfigured; private QueueItemState _ItemState = QueueItemState.NeverStarted; private DateTime _LastRunTime = DateTime.UtcNow; + private double _LastRunDuration = 0; private DateTime _LastFinishTime { get @@ -61,7 +62,9 @@ namespace gaseous_server public QueueItemState ItemState => _ItemState; public DateTime LastRunTime => _LastRunTime; public DateTime LastFinishTime => _LastFinishTime; - public DateTime NextRunTime { + public double LastRunDuration => _LastRunDuration; + public DateTime NextRunTime + { get { return LastRunTime.AddMinutes(Interval); @@ -85,6 +88,8 @@ namespace gaseous_server public bool RemoveWhenStopped => _RemoveWhenStopped; public bool IsBlocked => _IsBlocked; public object? Options { get; set; } = null; + public string CurrentState { get; set; } = ""; + public string CurrentStateProgress { get; set; } = ""; public List Blocks => _Blocks; public void Execute() @@ -107,8 +112,11 @@ namespace gaseous_server { case QueueItemType.SignatureIngestor: Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Signature Ingestor"); - SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor(); - + SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor + { + CallingQueueItem = this + }; + Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing TOSEC files"); tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "TOSEC"), gaseous_signature_parser.parser.SignatureParser.TOSEC); @@ -124,7 +132,10 @@ namespace gaseous_server case QueueItemType.TitleIngestor: Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor"); - Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory); + Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory) + { + CallingQueueItem = this + }; Classes.ImportGame.DeleteOrphanedDirectories(Config.LibraryConfiguration.LibraryImportDirectory); @@ -134,7 +145,11 @@ namespace gaseous_server case QueueItemType.MetadataRefresh: Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher"); - Classes.MetadataManagement.RefreshMetadata(_ForceExecute); + Classes.MetadataManagement metadataManagement = new MetadataManagement + { + CallingQueueItem = this + }; + metadataManagement.RefreshMetadata(_ForceExecute); _SaveLastRunTime = true; @@ -150,7 +165,11 @@ namespace gaseous_server case QueueItemType.LibraryScan: Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner"); - Classes.ImportGame.LibraryScan(); + Classes.ImportGame import = new ImportGame + { + CallingQueueItem = this + }; + import.LibraryScan(); _SaveLastRunTime = true; @@ -158,7 +177,11 @@ namespace gaseous_server case QueueItemType.Rematcher: Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Rematch"); - Classes.ImportGame.Rematcher(_ForceExecute); + Classes.ImportGame importRematch = new ImportGame + { + CallingQueueItem = this + }; + importRematch.Rematcher(_ForceExecute); _SaveLastRunTime = true; @@ -181,7 +204,10 @@ namespace gaseous_server case QueueItemType.Maintainer: Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Maintenance"); - Classes.Maintenance.RunMaintenance(); + Classes.Maintenance maintenance = new Maintenance{ + CallingQueueItem = this + }; + maintenance.RunMaintenance(); break; } @@ -196,8 +222,9 @@ namespace gaseous_server _ForceExecute = false; _ItemState = QueueItemState.Stopped; _LastFinishTime = DateTime.UtcNow; + _LastRunDuration = Math.Round((DateTime.UtcNow - _LastRunTime).TotalSeconds, 2); - Logging.Log(Logging.LogType.Information, "Timered Event", "Total " + _ItemType + " run time = " + (DateTime.UtcNow - _LastRunTime).TotalSeconds); + Logging.Log(Logging.LogType.Information, "Timered Event", "Total " + _ItemType + " run time = " + _LastRunDuration); } } } diff --git a/gaseous-server/wwwroot/pages/settings/system.html b/gaseous-server/wwwroot/pages/settings/system.html index 1cf23ef..d653d85 100644 --- a/gaseous-server/wwwroot/pages/settings/system.html +++ b/gaseous-server/wwwroot/pages/settings/system.html @@ -25,12 +25,13 @@

Signatures

- + setInterval(SystemSignaturesStatus, 300000); +