sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Signature ingestor database update is now a background task (#121)

Browse Source 
* Updated background task code to support options and self clearing

* Moved background safe database upgrade code to a background task
This commit is contained in:
Michael Green
2023-09-20 00:35:24 +10:00
committed by GitHub
parent 60fab488a2
commit 7eb418d6a2
7 changed files with 239 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
gaseous-server/Classes/Collections.cs
View File

@@ -171,13 +171,10 @@ namespace gaseous_server.Classes
db.ExecuteCMD(sql, dbDict);
// start background task
foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
{
if (qi.ItemType == ProcessQueue.QueueItemType.CollectionCompiler) {
qi.ForceExecute();
break;
}
}
ProcessQueue.QueueItem queueItem = new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.CollectionCompiler, 1, false, true);
queueItem.Options = Id;
queueItem.ForceExecute();
ProcessQueue.QueueItems.Add(queueItem);
}
}
@@ -364,13 +361,11 @@ namespace gaseous_server.Classes
return collectionContents;
}
public static void CompileCollections()
public static void CompileCollections(long CollectionId)
{
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
List<CollectionItem> collectionItems = GetCollections();
foreach (CollectionItem collectionItem in collectionItems)
{
CollectionItem collectionItem = GetCollection(CollectionId);
if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild)
{
Logging.Log(Logging.LogType.Information, "Collections", "Beginning build of collection: " + collectionItem.Name);
@@ -536,7 +531,6 @@ namespace gaseous_server.Classes
}
}
}
}
private static CollectionItem BuildCollectionItem(DataRow row) {
string strPlatforms = (string)Common.ReturnValueIfNull(row["Platforms"], "[ ]");

54
gaseous-server/ProcessQueue.cs
View File

@@ -9,34 +9,51 @@ namespace gaseous_server
public class QueueItem
{
public QueueItem(QueueItemType ItemType, int ExecutionInterval, bool AllowManualStart = true)
public QueueItem(QueueItemType ItemType, int ExecutionInterval, bool AllowManualStart = true, bool RemoveWhenStopped = false)
{
_ItemType = ItemType;
_ItemState = QueueItemState.NeverStarted;
_LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval);
_LastRunTime = DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
_Interval = ExecutionInterval;
_AllowManualStart = AllowManualStart;
_RemoveWhenStopped = RemoveWhenStopped;
}
public QueueItem(QueueItemType ItemType, int ExecutionInterval, List<QueueItemType> Blocks, bool AllowManualStart = true)
public QueueItem(QueueItemType ItemType, int ExecutionInterval, List<QueueItemType> Blocks, bool AllowManualStart = true, bool RemoveWhenStopped = false)
{
_ItemType = ItemType;
_ItemState = QueueItemState.NeverStarted;
_LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval);
_LastRunTime = DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
_Interval = ExecutionInterval;
_AllowManualStart = AllowManualStart;
_RemoveWhenStopped = RemoveWhenStopped;
_Blocks = Blocks;
}
private QueueItemType _ItemType = QueueItemType.NotConfigured;
private QueueItemState _ItemState = QueueItemState.NeverStarted;
private DateTime _LastRunTime = DateTime.UtcNow;
private DateTime _LastFinishTime = DateTime.UtcNow;
private DateTime _LastFinishTime
{
get
{
return DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
}
set
{
if (_SaveLastRunTime == true)
{
Config.SetSetting("LastRun_" + _ItemType.ToString(), value.ToString("yyyy-MM-ddThh:mm:ssZ"));
}
}
}
private bool _SaveLastRunTime = false;
private int _Interval = 0;
private string _LastResult = "";
private string? _LastError = null;
private bool _ForceExecute = false;
private bool _AllowManualStart = true;
private bool _RemoveWhenStopped = false;
private List<QueueItemType> _Blocks = new List<QueueItemType>();
public QueueItemType ItemType => _ItemType;
@@ -54,6 +71,8 @@ namespace gaseous_server
public string? LastError => _LastError;
public bool Force => _ForceExecute;
public bool AllowManualStart => _AllowManualStart;
public bool RemoveWhenStopped => _RemoveWhenStopped;
public object? Options { get; set; } = null;
public List<QueueItemType> Blocks => _Blocks;
public void Execute()
@@ -87,32 +106,52 @@ namespace gaseous_server
Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME MESS files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess);
_SaveLastRunTime = true;
break;
case QueueItemType.TitleIngestor:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor");
Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory);
_SaveLastRunTime = true;
break;
case QueueItemType.MetadataRefresh:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher");
Classes.MetadataManagement.RefreshMetadata(true);
_SaveLastRunTime = true;
break;
case QueueItemType.OrganiseLibrary:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Organiser");
Classes.ImportGame.OrganiseLibrary();
_SaveLastRunTime = true;
break;
case QueueItemType.LibraryScan:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner");
Classes.ImportGame.LibraryScan();
_SaveLastRunTime = true;
break;
case QueueItemType.CollectionCompiler:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
Classes.Collections.CompileCollections();
Classes.Collections.CompileCollections((long)Options);
break;
case QueueItemType.BackgroundDatabaseUpgrade:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Background Upgrade");
gaseous_tools.DatabaseMigration.UpgradeScriptBackgroundTasks();
break;
}
}
catch (Exception ex)
@@ -143,7 +182,8 @@ namespace gaseous_server
MetadataRefresh,
OrganiseLibrary,
LibraryScan,
CollectionCompiler
CollectionCompiler,
BackgroundDatabaseUpgrade
}
public enum QueueItemState

9
gaseous-server/Program.cs
View File

@@ -170,7 +170,14 @@ ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.OrganiseLibrary
})
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.CollectionCompiler, 5, false));
// kick off any delayed upgrade tasks
// run 1002 background updates in the background on every start
DatabaseMigration.BackgroundUpgradeTargetSchemaVersions.Add(1002);
// start the task
ProcessQueue.QueueItem queueItem = new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.BackgroundDatabaseUpgrade, 1, false, true);
queueItem.ForceExecute();
ProcessQueue.QueueItems.Add(queueItem);
// start the app
app.Run();

8
gaseous-server/Timer.cs
View File

@@ -33,9 +33,15 @@ namespace gaseous_server
//_logger.LogInformation(
// "Timed Hosted Service is working. Count: {Count}", count);
foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) {
List<ProcessQueue.QueueItem> ActiveList = new List<ProcessQueue.QueueItem>();
ActiveList.AddRange(ProcessQueue.QueueItems);
foreach (ProcessQueue.QueueItem qi in ActiveList) {
if ((DateTime.UtcNow > qi.NextRunTime || qi.Force == true) && CheckProcessBlockList(qi) == true) {
qi.Execute();
if (qi.RemoveWhenStopped == true && qi.ItemState == ProcessQueue.QueueItemState.Stopped)
{
ProcessQueue.QueueItems.Remove(qi);
}
}
}
}

7
gaseous-server/wwwroot/pages/dialogs/collectionedit.html
View File

@@ -394,7 +394,10 @@
}
modalAlwaysInclude = alwaysInclude;
console.log(JSON.stringify(modalAlwaysInclude));
if (!alwaysInclude) {
alwaysInclude = [];
}
var item = {
"name": document.getElementById('collection_name').value,
@@ -414,6 +417,8 @@
"alwaysInclude": alwaysInclude
}
console.log("Item: " + JSON.stringify(item));
return item;
}

16
gaseous-server/wwwroot/pages/settings/system.html
View File

@@ -52,7 +52,10 @@
itemTypeName = "Library scan";
break;
case 'CollectionCompiler':
itemTypeName = "Compress collections";
itemTypeName = "Compress collection id: " + result[i].options;
break;
case 'BackgroundDatabaseUpgrade':
itemTypeName = "Background database upgrade";
break;
default:
itemTypeName = result[i].itemType;
@@ -80,17 +83,24 @@
break;
}
var itemInterval = result[i].interval;
var nextRunTime = moment(result[i].nextRunTime).fromNow();
var startButton = '';
if (result[i].allowManualStart == true && result[i].itemState != "Running") {
startButton = "<span id='startProcess' class='romstart' onclick='StartProcess(\"" + result[i].itemType + "\");'>Start</span>";
}
if (result[i].allowManualStart == false && result[i].removeWhenStopped == true) {
itemInterval = '';
nextRunTime = '';
}
var newRow = [
itemTypeName,
itemStateName,
result[i].interval,
itemInterval,
itemLastStart,
moment(result[i].nextRunTime).fromNow(),
nextRunTime,
startButton
];
newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell'));

29
gaseous-tools/DatabaseMigration.cs
View File

@@ -5,26 +5,43 @@ namespace gaseous_tools
{
public static class DatabaseMigration
{
public static void PreUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType) {
public static List<int> BackgroundUpgradeTargetSchemaVersions = new List<int>();
public static void PreUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType)
{
}
public static void PostUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType) {
public static void PostUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType)
{
switch(DatabaseType)
{
case gaseous_tools.Database.databaseType.MySql:
case Database.databaseType.MySql:
switch (TargetSchemaVersion)
{
case 1002:
// this is a safe background task
BackgroundUpgradeTargetSchemaVersions.Add(1002);
break;
}
break;
}
}
public static void UpgradeScriptBackgroundTasks()
{
foreach (int TargetSchemaVersion in BackgroundUpgradeTargetSchemaVersions)
{
switch (TargetSchemaVersion)
{
case 1002:
MySql_1002_MigrateMetadataVersion();
break;
}
break;
}
}
private static void MySql_1002_MigrateMetadataVersion() {
public static void MySql_1002_MigrateMetadataVersion() {
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();