sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve background task progress feedback (#228)

Browse Source 
* Include a last run duration field for background tasks

* Improved background task progress feedback
This commit is contained in:
Michael Green
2023-12-12 17:42:40 +11:00
committed by GitHub
parent 32051493a8
commit 789ec7fc17
7 changed files with 151 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
gaseous-server/Classes/ImportGames.cs
View File

@@ -1,17 +1,19 @@
using System;
using System.Data;
using System.IO.Compression;
using System.Security.Authentication;
using System.Security.Policy;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using gaseous_server.Classes.Metadata;
using IGDB.Models;
using NuGet.Common;
using NuGet.LibraryModel;
using static gaseous_server.Classes.Metadata.Games;
namespace gaseous_server.Classes
{
public class ImportGames
public class ImportGames : QueueItemStatus
{
public ImportGames(string ImportPath)
{
@@ -21,9 +23,15 @@ namespace gaseous_server.Classes
string[] importContents_Directories = Directory.GetDirectories(ImportPath);
// import files first
int importCount = 1;
foreach (string importContent in importContents_Files) {
SetStatus(importCount, importContents_Files.Length, "Importing file: " + importContent);
ImportGame.ImportGameFile(importContent, null);
importCount += 1;
}
ClearStatus();
// import sub directories
foreach (string importDir in importContents_Directories) {
@@ -40,7 +48,7 @@ namespace gaseous_server.Classes
}
public class ImportGame
public class ImportGame : QueueItemStatus
{
public static void ImportGameFile(string GameFileImportPath, IGDB.Models.Platform? OverridePlatform)
{
@@ -600,7 +608,7 @@ namespace gaseous_server.Classes
}
}
public static void LibraryScan()
public void LibraryScan()
{
foreach (GameLibrary.LibraryItem library in GameLibrary.GetLibraries)
{
@@ -645,8 +653,10 @@ namespace gaseous_server.Classes
// search for files in the library that aren't in the database
Logging.Log(Logging.LogType.Information, "Library Scan", "Looking for orphaned library files to add");
string[] LibraryFiles = Directory.GetFiles(library.Path, "*.*", SearchOption.AllDirectories);
int StatusCount = 0;
foreach (string LibraryFile in LibraryFiles)
{
SetStatus(StatusCount, LibraryFiles.Length, "Processing file " + LibraryFile);
if (!Common.SkippableFiles.Contains<string>(Path.GetFileName(LibraryFile), StringComparer.OrdinalIgnoreCase))
{
Common.hashObject LibraryFileHash = new Common.hashObject(LibraryFile);
@@ -702,6 +712,7 @@ namespace gaseous_server.Classes
}
}
}
ClearStatus();
sql = "SELECT * FROM Games_Roms WHERE LibraryId=@libraryid ORDER BY `name`";
dtRoms = db.ExecuteCMD(sql, dbDict);
@@ -746,7 +757,7 @@ namespace gaseous_server.Classes
}
}
public static void Rematcher(bool ForceExecute = false)
public void Rematcher(bool ForceExecute = false)
{
// rescan all titles with an unknown platform or title and see if we can get a match
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan starting");
@@ -764,8 +775,11 @@ namespace gaseous_server.Classes
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("lastmatchattemptdate", DateTime.UtcNow.AddDays(-7));
DataTable data = db.ExecuteCMD(sql, dbDict);
int StatusCount = -0;
foreach (DataRow row in data.Rows)
{
SetStatus(StatusCount, data.Rows.Count, "Running rematcher");
// get library
GameLibrary.LibraryItem library = GameLibrary.GetLibrary((int)row["LibraryId"]);
@@ -800,9 +814,13 @@ namespace gaseous_server.Classes
dbLastAttemptDict.Add("id", romId);
dbLastAttemptDict.Add("lastmatchattemptdate", DateTime.UtcNow);
db.ExecuteCMD(attemptSql, dbLastAttemptDict);
StatusCount += 1;
}
ClearStatus();
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan completed");
ClearStatus();
}
}
}

12
gaseous-server/Classes/Maintenance.cs
View File

@@ -5,11 +5,11 @@ using Microsoft.VisualStudio.Web.CodeGeneration;
namespace gaseous_server.Classes
{
public class Maintenance
public class Maintenance : QueueItemStatus
{
const int MaxFileAge = 30;
public static void RunMaintenance()
public void RunMaintenance()
{
// delete files and directories older than 7 days in PathsToClean
List<string> PathsToClean = new List<string>();
@@ -49,8 +49,11 @@ namespace gaseous_server.Classes
string sql = "SHOW TABLES;";
DataTable tables = db.ExecuteCMD(sql);
int StatusCounter = 1;
foreach (DataRow row in tables.Rows)
{
SetStatus(StatusCounter, tables.Rows.Count, "Optimising table " + row[0].ToString());
sql = "OPTIMIZE TABLE " + row[0].ToString();
DataTable response = db.ExecuteCMD(sql);
foreach (DataRow responseRow in response.Rows)
@@ -60,9 +63,12 @@ namespace gaseous_server.Classes
{
retVal += responseRow.ItemArray[i] + "; ";
}
Logging.Log(Logging.LogType.Information, "Maintenance", "Optimise table " + row[0].ToString() + ": " + retVal);
Logging.Log(Logging.LogType.Information, "Maintenance", "(" + StatusCounter + "/" + tables.Rows.Count + "): Optimise table " + row[0].ToString() + ": " + retVal);
}
StatusCounter += 1;
}
ClearStatus();
}
}
}

23
gaseous-server/Classes/MetadataManagement.cs
View File

@@ -4,30 +4,39 @@ using gaseous_server.Models;
namespace gaseous_server.Classes
{
public class MetadataManagement
public class MetadataManagement : QueueItemStatus
{
public static void RefreshMetadata(bool forceRefresh = false)
public void RefreshMetadata(bool forceRefresh = false)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "";
DataTable dt = new DataTable();
// disabling forceRefresh
forceRefresh = false;
// update platforms
sql = "SELECT Id, `Name` FROM Platform;";
dt = db.ExecuteCMD(sql);
int StatusCounter = 1;
foreach (DataRow dr in dt.Rows)
{
SetStatus(StatusCounter, dt.Rows.Count, "Refreshing metadata for platform " + dr["name"]);
try
{
Logging.Log(Logging.LogType.Information, "Metadata Refresh", "Refreshing metadata for platform " + dr["name"] + " (" + dr["id"] + ")");
Logging.Log(Logging.LogType.Information, "Metadata Refresh", "(" + StatusCounter + "/" + dt.Rows.Count + "): Refreshing metadata for platform " + dr["name"] + " (" + dr["id"] + ")");
Metadata.Platforms.GetPlatform((long)dr["id"], true);
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Critical, "Metadata Refresh", "An error occurred while refreshing metadata for " + dr["name"], ex);
}
StatusCounter += 1;
}
ClearStatus();
// update games
if (forceRefresh == true)
@@ -42,18 +51,24 @@ namespace gaseous_server.Classes
}
dt = db.ExecuteCMD(sql);
StatusCounter = 1;
foreach (DataRow dr in dt.Rows)
{
SetStatus(StatusCounter, dt.Rows.Count, "Refreshing metadata for game " + dr["name"]);
try
{
Logging.Log(Logging.LogType.Information, "Metadata Refresh", "Refreshing metadata for game " + dr["name"] + " (" + dr["id"] + ")");
Logging.Log(Logging.LogType.Information, "Metadata Refresh", "(" + StatusCounter + "/" + dt.Rows.Count + "): Refreshing metadata for game " + dr["name"] + " (" + dr["id"] + ")");
Metadata.Games.GetGame((long)dr["id"], true, false, forceRefresh);
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Critical, "Metadata Refresh", "An error occurred while refreshing metadata for " + dr["name"], ex);
}
StatusCounter += 1;
}
ClearStatus();
}
}
}

41
gaseous-server/Classes/QueueItemStatus.cs Normal file
View File

@@ -0,0 +1,41 @@
namespace gaseous_server.Classes
{
public class QueueItemStatus
{
internal ProcessQueue.QueueItem? CallingQueueItem = null;
private int _CurrentItemNumber = 0;
private int _MaxItemsNumber = 0;
private string _StatusText = "";
public int CurrentItemNumber => _CurrentItemNumber;
public int MaxItemsNumber => _MaxItemsNumber;
public string StatusText => _StatusText;
public void SetStatus(int CurrentItemNumber, int MaxItemsNumber, string StatusText)
{
this._CurrentItemNumber = CurrentItemNumber;
this._MaxItemsNumber = MaxItemsNumber;
this._StatusText = StatusText;
if (CallingQueueItem != null)
{
CallingQueueItem.CurrentState = _CurrentItemNumber + " of " + _MaxItemsNumber + ": " + _StatusText;
CallingQueueItem.CurrentStateProgress = _CurrentItemNumber + " of " + _MaxItemsNumber;
}
}
public void ClearStatus()
{
this._CurrentItemNumber = 0;
this._MaxItemsNumber = 0;
this._StatusText = "";
if (CallingQueueItem != null)
{
CallingQueueItem.CurrentState = "";
CallingQueueItem.CurrentStateProgress = "";
}
}
}
}

5
gaseous-server/Classes/SignatureIngestors/XML.cs
View File

@@ -6,7 +6,7 @@ using System.Data;
namespace gaseous_server.SignatureIngestors.XML
{
public class XMLIngestor
public class XMLIngestor : QueueItemStatus
{
public void Import(string SearchPath, gaseous_signature_parser.parser.SignatureParser XMLType)
{
@@ -31,6 +31,8 @@ namespace gaseous_server.SignatureIngestors.XML
{
string XMLFile = PathContents[i];
SetStatus(i + 1, PathContents.Length, "Processing signature file: " + XMLFile);
// check xml file md5
Common.hashObject hashObject = new Common.hashObject(XMLFile);
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
@@ -247,6 +249,7 @@ namespace gaseous_server.SignatureIngestors.XML
Logging.Log(Logging.LogType.Debug, "Signature Ingestor - XML", "Rejecting already imported file: " + XMLFile);
}
}
ClearStatus();
}
}
}

43
gaseous-server/ProcessQueue.cs
View File

@@ -33,6 +33,7 @@ namespace gaseous_server
private QueueItemType _ItemType = QueueItemType.NotConfigured;
private QueueItemState _ItemState = QueueItemState.NeverStarted;
private DateTime _LastRunTime = DateTime.UtcNow;
private double _LastRunDuration = 0;
private DateTime _LastFinishTime
{
get
@@ -61,7 +62,9 @@ namespace gaseous_server
public QueueItemState ItemState => _ItemState;
public DateTime LastRunTime => _LastRunTime;
public DateTime LastFinishTime => _LastFinishTime;
public DateTime NextRunTime {
public double LastRunDuration => _LastRunDuration;
public DateTime NextRunTime
{
get
{
return LastRunTime.AddMinutes(Interval);
@@ -85,6 +88,8 @@ namespace gaseous_server
public bool RemoveWhenStopped => _RemoveWhenStopped;
public bool IsBlocked => _IsBlocked;
public object? Options { get; set; } = null;
public string CurrentState { get; set; } = "";
public string CurrentStateProgress { get; set; } = "";
public List<QueueItemType> Blocks => _Blocks;
public void Execute()
@@ -107,7 +112,10 @@ namespace gaseous_server
{
case QueueItemType.SignatureIngestor:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Signature Ingestor");
SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor();
SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor
{
CallingQueueItem = this
};
Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing TOSEC files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "TOSEC"), gaseous_signature_parser.parser.SignatureParser.TOSEC);
@@ -124,7 +132,10 @@ namespace gaseous_server
case QueueItemType.TitleIngestor:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor");
Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory);
Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory)
{
CallingQueueItem = this
};
Classes.ImportGame.DeleteOrphanedDirectories(Config.LibraryConfiguration.LibraryImportDirectory);
@@ -134,7 +145,11 @@ namespace gaseous_server
case QueueItemType.MetadataRefresh:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher");
Classes.MetadataManagement.RefreshMetadata(_ForceExecute);
Classes.MetadataManagement metadataManagement = new MetadataManagement
{
CallingQueueItem = this
};
metadataManagement.RefreshMetadata(_ForceExecute);
_SaveLastRunTime = true;
@@ -150,7 +165,11 @@ namespace gaseous_server
case QueueItemType.LibraryScan:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner");
Classes.ImportGame.LibraryScan();
Classes.ImportGame import = new ImportGame
{
CallingQueueItem = this
};
import.LibraryScan();
_SaveLastRunTime = true;
@@ -158,7 +177,11 @@ namespace gaseous_server
case QueueItemType.Rematcher:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Rematch");
Classes.ImportGame.Rematcher(_ForceExecute);
Classes.ImportGame importRematch = new ImportGame
{
CallingQueueItem = this
};
importRematch.Rematcher(_ForceExecute);
_SaveLastRunTime = true;
@@ -181,7 +204,10 @@ namespace gaseous_server
case QueueItemType.Maintainer:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Maintenance");
Classes.Maintenance.RunMaintenance();
Classes.Maintenance maintenance = new Maintenance{
CallingQueueItem = this
};
maintenance.RunMaintenance();
break;
}
@@ -196,8 +222,9 @@ namespace gaseous_server
_ForceExecute = false;
_ItemState = QueueItemState.Stopped;
_LastFinishTime = DateTime.UtcNow;
_LastRunDuration = Math.Round((DateTime.UtcNow - _LastRunTime).TotalSeconds, 2);
Logging.Log(Logging.LogType.Information, "Timered Event", "Total " + _ItemType + " run time = " + (DateTime.UtcNow - _LastRunTime).TotalSeconds);
Logging.Log(Logging.LogType.Information, "Timered Event", "Total " + _ItemType + " run time = " + _LastRunDuration);
}
}
}

30
gaseous-server/wwwroot/pages/settings/system.html
View File

@@ -25,12 +25,13 @@
<h3>Signatures</h3>
<div id="system_signatures"></div>
<script type="text/javascript">function SystemLoadStatus() {
<script type="text/javascript">
function SystemLoadStatus() {
ajaxCall('/api/v1.1/BackgroundTasks', 'GET', function (result) {
var newTable = document.createElement('table');
newTable.className = 'romtable';
newTable.setAttribute('cellspacing', 0);
newTable.appendChild(createTableRow(true, ['Task', 'Status', 'Interval', 'Last Run Time', 'Next Run Time', '']));
newTable.appendChild(createTableRow(true, ['Task', 'Status', 'Interval', 'Last Run Start', 'Last Run Duration (seconds)', 'Next Run Start', '']));
if (result) {
for (var i = 0; i < result.length; i++) {
@@ -46,15 +47,19 @@
break;
case 'Stopped':
itemStateName = "Stopped";
itemLastStart = moment(result[i].lastRunTime).fromNow();
itemLastStart = moment(result[i].lastRunTime).format("YYYY-MM-DD h:mm:ss a");
break;
case 'Running':
itemStateName = "Running";
itemLastStart = moment(result[i].lastRunTime).fromNow();
var progressPercent = "";
if (result[i].currentStateProgress) {
progressPercent = " (" + result[i].currentStateProgress + ")";
}
itemStateName = "Running" + progressPercent;
itemLastStart = moment(result[i].lastRunTime).format("YYYY-MM-DD h:mm:ss a");
break;
default:
itemStateName = "Unknown status";
itemLastStart = moment(result[i].lastRunTime).fromNow();
itemLastStart = moment(result[i].lastRunTime).format("YYYY-MM-DD h:mm:ss a");
break;
}
} else {
@@ -63,7 +68,7 @@
}
var itemInterval = result[i].interval;
var nextRunTime = moment(result[i].nextRunTime).fromNow();
var nextRunTime = moment(result[i].nextRunTime).format("YYYY-MM-DD h:mm:ss a");
var startButton = '';
if (userProfile.roles.includes("Admin")) {
if (result[i].allowManualStart == true && result[i].itemState != "Running") {
@@ -81,6 +86,7 @@
itemStateName,
itemInterval,
itemLastStart,
result[i].lastRunDuration,
nextRunTime,
startButton
];
@@ -176,6 +182,9 @@
}
function BuildLibraryStatisticsBar(TargetObject, TargetObjectLegend, LibraryStatistics, LibrarySize) {
TargetObject.innerHTML = '';
TargetObjectLegend.innerHTML = '';
var newTable = document.createElement('table');
newTable.setAttribute('cellspacing', 0);
newTable.setAttribute('style', 'width: 100%; height: 10px;');
@@ -239,8 +248,9 @@
}
SystemLoadStatus();
setInterval(SystemLoadStatus, 30000);
setInterval(SystemLoadStatus, 3000);
SystemLoadSystemStatus();
setInterval(SystemLoadStatus, 60000);
setInterval(SystemLoadSystemStatus, 60000);
SystemSignaturesStatus();
setInterval(SystemSignaturesStatus, 300000);</script>
setInterval(SystemSignaturesStatus, 300000);
</script>