feat: scaffolding for background tasks complete

gaseous-server/Controllers/BackgroundTasksController.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+
+namespace gaseous_server.Controllers
+{
+    [ApiController]
+    [Route("api/v1/[controller]")]
+    public class BackgroundTasksController : Controller
+    {
+        [HttpGet]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        public List<ProcessQueue.QueueItem> GetQueue()
+        {
+            return ProcessQueue.QueueItems;
+        }
+
+        [HttpGet]
+        [Route("{TaskType}")]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        [ProducesResponseType(StatusCodes.Status404NotFound)]
+        public ActionResult<ProcessQueue.QueueItem> ForceRun(ProcessQueue.QueueItemType TaskType, Boolean ForceRun)
+        {
+            foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
+            {
+                if (TaskType == qi.ItemType)
+                {
+                    if (ForceRun == true)
+                    {
+                        qi.ForceExecute();
+                    }
+                    return qi;
+                }
+            }
+
+            return NotFound();
+        }
+    }
+}

gaseous-server/Controllers/SignaturesController.cs

@@ -20,13 +20,14 @@ namespace gaseous_server.Controllers
         /// </summary>
         /// <returns>Number of sources, publishers, games, and rom signatures in the database</returns>
         [HttpGet]
+        [ProducesResponseType(StatusCodes.Status200OK)]
         public Models.Signatures_Status Status()
         {
             return new Models.Signatures_Status();
         }
 
         [HttpGet]
-        [Route("api/[controller]/[action]")]
+        [ProducesResponseType(StatusCodes.Status200OK)]
         public List<Models.Signatures_Games> GetSignature(string md5 = "", string sha1 = "")
         {
             if (md5.Length > 0)

gaseous-server/ProcessQueue.cs

@@ -0,0 +1,107 @@
+using System;
+using gaseous_tools;
+
+namespace gaseous_server
+{
+	public static class ProcessQueue
+	{
+        public static List<QueueItem> QueueItems = new List<QueueItem>();
+
+        public class QueueItem
+        {
+            public QueueItem(QueueItemType ItemType, int ExecutionInterval)
+            {
+                _ItemType = ItemType;
+                _ItemState = QueueItemState.NeverStarted;
+                _LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval);
+                _Interval = ExecutionInterval;
+            }
+
+            private QueueItemType _ItemType = QueueItemType.NotConfigured;
+            private QueueItemState _ItemState = QueueItemState.NeverStarted;
+            private DateTime _LastRunTime = DateTime.UtcNow;
+            private DateTime _LastFinishTime = DateTime.UtcNow;
+            private int _Interval = 0;
+            private string _LastResult = "";
+            private Exception? _LastError = null;
+            private bool _ForceExecute = false;
+
+            public QueueItemType ItemType => _ItemType;
+            public QueueItemState ItemState => _ItemState;
+            public DateTime LastRunTime => _LastRunTime;
+            public DateTime LastFinishTime => _LastFinishTime;
+            public DateTime NextRunTime {
+                get
+                {
+                    return LastRunTime.AddMinutes(Interval);
+                }
+            }
+            public int Interval => _Interval;
+            public string LastResult => _LastResult;
+            public Exception? LastError => _LastError;
+            public bool Force => _ForceExecute;
+
+            public void Execute()
+            {
+                if (_ItemState != QueueItemState.Disabled)
+                {
+                    if ((DateTime.UtcNow > NextRunTime || _ForceExecute == true) && _ItemState != QueueItemState.Running)
+                    {
+                        // we can run - do some setup before we start processing
+                        _LastRunTime = DateTime.UtcNow;
+                        _ItemState = QueueItemState.Running;
+                        _LastResult = "";
+                        _LastError = null;
+                        _ForceExecute = false;
+
+                        Logging.Log(Logging.LogType.Information, "Timered Event", "Executing " + _ItemType);
+
+                        try
+                        {
+                            switch (_ItemType)
+                            {
+                                case QueueItemType.SignatureIngestor:
+                                    Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Signature Ingestor");
+                                    break;
+
+                                case QueueItemType.TitleIngestor:
+                                    Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Title Ingestor");
+                                    break;
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            Logging.Log(Logging.LogType.Warning, "Timered Event", "An error occurred", ex);
+                            _LastResult = "";
+                            _LastError = ex;
+                        }
+
+                        _ItemState = QueueItemState.Stopped;
+                        _LastFinishTime = DateTime.UtcNow;
+                    }
+                }
+            }
+
+            public void ForceExecute()
+            {
+                _ForceExecute = true;
+            }
+        }
+
+        public enum QueueItemType
+        {
+            NotConfigured,
+            SignatureIngestor,
+            TitleIngestor
+        }
+
+        public enum QueueItemState
+        {
+            NeverStarted,
+            Running,
+            Stopped,
+            Disabled
+        }
+    }
+}
+

gaseous-server/Program.cs

@@ -2,20 +2,31 @@
 using gaseous_server;
 using gaseous_tools;
 
+Logging.Log(Logging.LogType.Information, "Startup", "Starting Gaseous Server");
+
 // set up db
 Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 db.InitDB();
 
+// set initial values
+Guid APIKey = Guid.NewGuid();
+if (Config.ReadSetting("API Key", "Test API Key") == "Test API Key")
+{
+    // it's a new api key save it
+    Logging.Log(Logging.LogType.Information, "Startup", "Setting initial API key");
+    Config.SetSetting("API Key", APIKey.ToString());
+}
+
 // set up server
 var builder = WebApplication.CreateBuilder(args);
 
 // Add services to the container.
-
 builder.Services.AddControllers().AddJsonOptions(x =>
 {
     // serialize enums as strings in api responses (e.g. Role)
     x.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter());
 });
+
 // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddSwaggerGen();
@@ -36,6 +47,9 @@ app.UseAuthorization();
 
 app.MapControllers();
 
+// add background tasks
+ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60));
+ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.TitleIngestor, 1));
+
 // start the app
 app.Run();
-

gaseous-server/Timer.cs

@@ -1,4 +1,5 @@
 using System;
+using gaseous_tools;
 
 namespace gaseous_server
 {
@@ -6,17 +7,18 @@ namespace gaseous_server
     public class TimedHostedService : IHostedService, IDisposable
     {
         private int executionCount = 0;
-        private readonly ILogger<TimedHostedService> _logger;
+        //private readonly ILogger<TimedHostedService> _logger;
         private Timer _timer;
 
-        public TimedHostedService(ILogger<TimedHostedService> logger)
+        //public TimedHostedService(ILogger<TimedHostedService> logger)
-        {
+        //{
-            _logger = logger;
+        //    _logger = logger;
-        }
+        //}
 
         public Task StartAsync(CancellationToken stoppingToken)
         {
-            _logger.LogInformation("Timed Hosted Service running.");
+            //_logger.LogInformation("Timed Hosted Service running.");
+            Logging.Log(Logging.LogType.Debug, "Background", "Starting background task monitor");
 
             _timer = new Timer(DoWork, null, TimeSpan.Zero,
                 TimeSpan.FromSeconds(5));
@@ -28,13 +30,20 @@ namespace gaseous_server
         {
             var count = Interlocked.Increment(ref executionCount);
 
-            _logger.LogInformation(
+            //_logger.LogInformation(
-                "Timed Hosted Service is working. Count: {Count}", count);
+            //    "Timed Hosted Service is working. Count: {Count}", count);
+
+            foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) {
+                if (DateTime.UtcNow > qi.NextRunTime || qi.Force == true) {
+                    qi.Execute();
+                }
+            }
         }
 
         public Task StopAsync(CancellationToken stoppingToken)
         {
-            _logger.LogInformation("Timed Hosted Service is stopping.");
+            //_logger.LogInformation("Timed Hosted Service is stopping.");
+            Logging.Log(Logging.LogType.Debug, "Background", "Stopping background task monitor");
 
             _timer?.Change(Timeout.Infinite, 0);
 

gaseous-server/gaseous-server.csproj

@@ -10,8 +10,9 @@
   <PropertyGroup Condition=" '$(RunConfiguration)' == 'https' " />
   <PropertyGroup Condition=" '$(RunConfiguration)' == 'http' " />
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.3" />
+    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.4" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="7.0.5" />
   </ItemGroup>
 
   <ItemGroup>
@@ -21,6 +22,9 @@
     <ProjectReference Include="..\gaseous-romsignatureobject\gaseous-romsignatureobject.csproj">
       <GlobalPropertiesToRemove></GlobalPropertiesToRemove>
     </ProjectReference>
+    <ProjectReference Include="..\gaseous-signature-ingestor\gaseous-signature-ingestor.csproj">
+      <GlobalPropertiesToRemove></GlobalPropertiesToRemove>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
     <None Remove="Controllers\" />

gaseous-tools/Config.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Data;
+using Google.Protobuf.WellKnownTypes;
 using Newtonsoft.Json;
 
 namespace gaseous_tools
@@ -40,6 +41,35 @@ namespace gaseous_tools
             }
         }
 
+        public static string LogPath
+        {
+            get
+            {
+                string logPath = Path.Combine(ConfigurationPath, "Logs");
+                if (!Directory.Exists(logPath)) {
+                    Directory.CreateDirectory(logPath);
+                }
+                return logPath;
+            }
+        }
+
+        public static string LogFilePath
+        {
+            get
+            {
+                string logPathName = Path.Combine(LogPath, "Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + ".txt");
+                return logPathName;
+            }
+        }
+
+        public static ConfigFile.Logging LoggingConfiguration
+        {
+            get
+            {
+                return _config.LoggingConfiguration;
+            }
+        }
+
         static Config()
         {
             if (_config == null)
@@ -61,8 +91,7 @@ namespace gaseous_tools
                     // no config file!
                     // use defaults and save
                     _config = new ConfigFile();
-                    string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, Newtonsoft.Json.Formatting.Indented);
+                    UpdateConfig();
-                    File.WriteAllText(ConfigurationFilePath, configRaw);
                 }
             }
 
@@ -73,7 +102,14 @@ namespace gaseous_tools
         public static void UpdateConfig()
         {
             // save any updates to the configuration
-            string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, Newtonsoft.Json.Formatting.Indented);
+            Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings
+            {
+                NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
+                Formatting = Newtonsoft.Json.Formatting.Indented
+            };
+            serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
+            string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, serializerSettings);
+
             if (File.Exists(ConfigurationFilePath_Backup))
             {
                 File.Delete(ConfigurationFilePath_Backup);
@@ -85,7 +121,7 @@ namespace gaseous_tools
             File.WriteAllText(ConfigurationFilePath, configRaw);
         }
 
-        private static string ReadSetting(string SettingName, string DefaultValue)
+        public static string ReadSetting(string SettingName, string DefaultValue)
         {
             Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
             string sql = "SELECT * FROM settings WHERE setting = @settingname";
@@ -93,6 +129,9 @@ namespace gaseous_tools
             dbDict.Add("settingname", SettingName);
             dbDict.Add("value", DefaultValue);
 
+            try
+            {
+                Logging.Log(Logging.LogType.Debug, "Database", "Reading setting '" + SettingName + "'");
                 DataTable dbResponse = db.ExecuteCMD(sql, dbDict);
                 if (dbResponse.Rows.Count == 0)
                 {
@@ -104,8 +143,14 @@ namespace gaseous_tools
                     return (string)dbResponse.Rows[0][0];
                 }
             }
+            catch (Exception ex)
+            {
+                Logging.Log(Logging.LogType.Critical, "Database", "Failed reading setting " + SettingName, ex);
+                throw;
+            }
+        }
 
-        private static void SetSetting(string SettingName, string Value)
+        public static void SetSetting(string SettingName, string Value)
         {
             Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
             string sql = "REPLACE INTO settings (setting, value) VALUES (@settingname, @value)";
@@ -113,8 +158,17 @@ namespace gaseous_tools
             dbDict.Add("settingname", SettingName);
             dbDict.Add("value", Value);
 
+            Logging.Log(Logging.LogType.Debug, "Database", "Storing setting '" + SettingName + "' to value: '" + Value + "'");
+            try
+            {
                 db.ExecuteCMD(sql, dbDict);
             }
+            catch (Exception ex)
+            {
+                Logging.Log(Logging.LogType.Critical, "Database", "Failed storing setting" + SettingName, ex);
+                throw;
+            }
+        }
 
         public class ConfigFile
         {
@@ -123,6 +177,8 @@ namespace gaseous_tools
             [JsonIgnore]
             public Library LibraryConfiguration = new Library();
 
+            public Logging LoggingConfiguration = new Logging();
+
             public class Database
             {
                 public string HostName = "localhost";
@@ -180,6 +236,19 @@ namespace gaseous_tools
                     }
                 }
             }
+
+            public class Logging
+            {
+                public bool DebugLogging = false;
+
+                public LoggingFormat LogFormat = Logging.LoggingFormat.Json;
+
+                public enum LoggingFormat
+                {
+                    Json,
+                    Text
+                }
+            }
         }
     }
 }

gaseous-tools/Database.cs

@@ -64,6 +64,7 @@ namespace gaseous_tools
 					// check if the database exists first - first run must have permissions to create a database
 					string sql = "CREATE DATABASE IF NOT EXISTS `" + Config.DatabaseConfiguration.DatabaseName + "`;";
 					Dictionary<string, object> dbDict = new Dictionary<string, object>();
+					Logging.Log(Logging.LogType.Information, "Database", "Creating database if it doesn't exist");
 					ExecuteCMD(sql, dbDict, 30, "server=" + Config.DatabaseConfiguration.HostName + ";port=" + Config.DatabaseConfiguration.Port + ";userid=" + Config.DatabaseConfiguration.UserName + ";password=" + Config.DatabaseConfiguration.Password);
 
 					// check if schema version table is in place - if not, create the schema version table
@@ -72,6 +73,7 @@ namespace gaseous_tools
 					if (SchemaVersionPresent.Rows.Count == 0)
 					{
                         // no schema table present - create it
+                        Logging.Log(Logging.LogType.Information, "Database", "Schema version table doesn't exist. Creating it.");
                         sql = "CREATE TABLE `schema_version` (`schema_version` INT NOT NULL, PRIMARY KEY (`schema_version`)); INSERT INTO `schema_version` (`schema_version`) VALUES (0);";
 						ExecuteCMD(sql, dbDict);
 					}
@@ -96,14 +98,17 @@ namespace gaseous_tools
 								if (SchemaVersion.Rows.Count == 0)
 								{
                                     // something is broken here... where's the table?
+                                    Logging.Log(Logging.LogType.Critical, "Database", "Schema table missing! This shouldn't happen!");
                                     throw new Exception("schema_version table is missing!");
 								}
 								else
 								{
 									int SchemaVer = (int)SchemaVersion.Rows[0][0];
+                                    Logging.Log(Logging.LogType.Information, "Database", "Schema version is " + SchemaVer);
                                     if (SchemaVer < i)
 									{
                                         // apply schema!
+                                        Logging.Log(Logging.LogType.Information, "Database", "Schema update available - applying");
                                         ExecuteCMD(dbScript, dbDict);
 
 										sql = "UPDATE schema_version SET schema_version=@schemaver";
@@ -115,6 +120,7 @@ namespace gaseous_tools
 							}
 						}
                     }
+                    Logging.Log(Logging.LogType.Information, "Database", "Database setup complete");
                     break;
 			}
 		}
@@ -161,6 +167,7 @@ namespace gaseous_tools
 			{
 				DataTable RetTable = new DataTable();
 
+                Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database");
                 MySqlConnection conn = new MySqlConnection(DBConn);
 				conn.Open();
 
@@ -178,12 +185,20 @@ namespace gaseous_tools
 
 				try
 				{
+                    Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'");
+					if (Parameters.Count > 0)
+					{
+						string dictValues = string.Join(";", Parameters.Select(x => string.Join("=", x.Key, x.Value)));
+						Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues);
+					}
                     RetTable.Load(cmd.ExecuteReader());
 				} catch (Exception ex) {
+					Logging.Log(Logging.LogType.Critical, "Database", "Error while executing '" + SQL + "'", ex);
 					Trace.WriteLine("Error executing " + SQL);
 					Trace.WriteLine("Full exception: " + ex.ToString());
 				}
 
+				Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection");
 				conn.Close();
 
 				return RetTable;

gaseous-tools/Logging.cs

@@ -0,0 +1,102 @@
+using System;
+namespace gaseous_tools
+{
+	public class Logging
+	{
+        static public void Log(LogType EventType, string Section, string Message, Exception? ExceptionValue = null)
+        {
+            LogItem logItem = new LogItem
+            {
+                EventTime = DateTime.UtcNow,
+                EventType = EventType,
+                Section = Section,
+                Message = Message,
+                ExceptionValue = ExceptionValue
+            };
+
+            bool AllowWrite = false;
+            if (EventType == LogType.Debug)
+            {
+                if (Config.LoggingConfiguration.DebugLogging == true)
+                {
+                    AllowWrite = true;
+                }
+            }
+            else
+            {
+                AllowWrite = true;
+            }
+
+            if (AllowWrite == true)
+            {
+                // console output
+                string TraceOutput = logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Section + ": " + logItem.Message;
+                if (logItem.ExceptionValue != null)
+                {
+                    TraceOutput += Environment.NewLine + logItem.ExceptionValue.ToString();
+                }
+                Console.WriteLine(TraceOutput);
+
+                StreamWriter LogFile = File.AppendText(Config.LogFilePath);
+                switch (Config.LoggingConfiguration.LogFormat)
+                {
+                    case Config.ConfigFile.Logging.LoggingFormat.Text:
+                        LogFile.WriteLine(TraceOutput);
+                        break;
+
+                    case Config.ConfigFile.Logging.LoggingFormat.Json:
+                        Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings
+                        {
+                            NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
+                            Formatting = Newtonsoft.Json.Formatting.Indented
+                        };
+                        serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
+                        string JsonOutput = Newtonsoft.Json.JsonConvert.SerializeObject(logItem, serializerSettings);
+                        LogFile.WriteLine(JsonOutput);
+                        break;
+                }
+                LogFile.Close();
+            }
+        }
+
+        public enum LogType
+        {
+            Information = 0,
+            Debug = 1,
+            Warning = 2,
+            Critical = 3
+        }
+
+        public class LogItem
+        {
+            public DateTime EventTime { get; set; }
+            public LogType EventType { get; set; }
+            private string _Section = "";
+            public string Section
+            {
+                get
+                {
+                    return _Section;
+                }
+                set
+                {
+                    _Section = value;
+                }
+            }
+            private string _Message = "";
+            public string Message
+            {
+                get
+                {
+                    return _Message;
+                }
+                set
+                {
+                    _Message = value;
+                }
+            }
+            public Exception? ExceptionValue { get; set; }
+        }
+    }
+}
+