diff --git a/gaseous-server/Controllers/BackgroundTasksController.cs b/gaseous-server/Controllers/BackgroundTasksController.cs new file mode 100644 index 0000000..062ffad --- /dev/null +++ b/gaseous-server/Controllers/BackgroundTasksController.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; + +namespace gaseous_server.Controllers +{ + [ApiController] + [Route("api/v1/[controller]")] + public class BackgroundTasksController : Controller + { + [HttpGet] + [ProducesResponseType(StatusCodes.Status200OK)] + public List GetQueue() + { + return ProcessQueue.QueueItems; + } + + [HttpGet] + [Route("{TaskType}")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public ActionResult ForceRun(ProcessQueue.QueueItemType TaskType, Boolean ForceRun) + { + foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) + { + if (TaskType == qi.ItemType) + { + if (ForceRun == true) + { + qi.ForceExecute(); + } + return qi; + } + } + + return NotFound(); + } + } +} \ No newline at end of file diff --git a/gaseous-server/Controllers/SignaturesController.cs b/gaseous-server/Controllers/SignaturesController.cs index e1f3e58..fd14de2 100644 --- a/gaseous-server/Controllers/SignaturesController.cs +++ b/gaseous-server/Controllers/SignaturesController.cs @@ -20,13 +20,14 @@ namespace gaseous_server.Controllers /// /// Number of sources, publishers, games, and rom signatures in the database [HttpGet] + [ProducesResponseType(StatusCodes.Status200OK)] public Models.Signatures_Status Status() { return new Models.Signatures_Status(); } [HttpGet] - [Route("api/[controller]/[action]")] + [ProducesResponseType(StatusCodes.Status200OK)] public List GetSignature(string md5 = "", string sha1 = "") { if (md5.Length > 0) diff --git a/gaseous-server/ProcessQueue.cs b/gaseous-server/ProcessQueue.cs new file mode 100644 index 0000000..e3e1c04 --- /dev/null +++ b/gaseous-server/ProcessQueue.cs @@ -0,0 +1,107 @@ +using System; +using gaseous_tools; + +namespace gaseous_server +{ + public static class ProcessQueue + { + public static List QueueItems = new List(); + + public class QueueItem + { + public QueueItem(QueueItemType ItemType, int ExecutionInterval) + { + _ItemType = ItemType; + _ItemState = QueueItemState.NeverStarted; + _LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval); + _Interval = ExecutionInterval; + } + + private QueueItemType _ItemType = QueueItemType.NotConfigured; + private QueueItemState _ItemState = QueueItemState.NeverStarted; + private DateTime _LastRunTime = DateTime.UtcNow; + private DateTime _LastFinishTime = DateTime.UtcNow; + private int _Interval = 0; + private string _LastResult = ""; + private Exception? _LastError = null; + private bool _ForceExecute = false; + + public QueueItemType ItemType => _ItemType; + public QueueItemState ItemState => _ItemState; + public DateTime LastRunTime => _LastRunTime; + public DateTime LastFinishTime => _LastFinishTime; + public DateTime NextRunTime { + get + { + return LastRunTime.AddMinutes(Interval); + } + } + public int Interval => _Interval; + public string LastResult => _LastResult; + public Exception? LastError => _LastError; + public bool Force => _ForceExecute; + + public void Execute() + { + if (_ItemState != QueueItemState.Disabled) + { + if ((DateTime.UtcNow > NextRunTime || _ForceExecute == true) && _ItemState != QueueItemState.Running) + { + // we can run - do some setup before we start processing + _LastRunTime = DateTime.UtcNow; + _ItemState = QueueItemState.Running; + _LastResult = ""; + _LastError = null; + _ForceExecute = false; + + Logging.Log(Logging.LogType.Information, "Timered Event", "Executing " + _ItemType); + + try + { + switch (_ItemType) + { + case QueueItemType.SignatureIngestor: + Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Signature Ingestor"); + break; + + case QueueItemType.TitleIngestor: + Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Title Ingestor"); + break; + } + } + catch (Exception ex) + { + Logging.Log(Logging.LogType.Warning, "Timered Event", "An error occurred", ex); + _LastResult = ""; + _LastError = ex; + } + + _ItemState = QueueItemState.Stopped; + _LastFinishTime = DateTime.UtcNow; + } + } + } + + public void ForceExecute() + { + _ForceExecute = true; + } + } + + public enum QueueItemType + { + NotConfigured, + SignatureIngestor, + TitleIngestor + } + + public enum QueueItemState + { + NeverStarted, + Running, + Stopped, + Disabled + } + } +} + diff --git a/gaseous-server/Program.cs b/gaseous-server/Program.cs index f0ae836..374a2de 100644 --- a/gaseous-server/Program.cs +++ b/gaseous-server/Program.cs @@ -2,20 +2,31 @@ using gaseous_server; using gaseous_tools; +Logging.Log(Logging.LogType.Information, "Startup", "Starting Gaseous Server"); + // set up db Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); db.InitDB(); +// set initial values +Guid APIKey = Guid.NewGuid(); +if (Config.ReadSetting("API Key", "Test API Key") == "Test API Key") +{ + // it's a new api key save it + Logging.Log(Logging.LogType.Information, "Startup", "Setting initial API key"); + Config.SetSetting("API Key", APIKey.ToString()); +} + // set up server var builder = WebApplication.CreateBuilder(args); // Add services to the container. - builder.Services.AddControllers().AddJsonOptions(x => { // serialize enums as strings in api responses (e.g. Role) x.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter()); }); + // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); @@ -36,6 +47,9 @@ app.UseAuthorization(); app.MapControllers(); +// add background tasks +ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60)); +ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.TitleIngestor, 1)); + // start the app app.Run(); - diff --git a/gaseous-server/Timer.cs b/gaseous-server/Timer.cs index 6d3a977..147898d 100644 --- a/gaseous-server/Timer.cs +++ b/gaseous-server/Timer.cs @@ -1,4 +1,5 @@ using System; +using gaseous_tools; namespace gaseous_server { @@ -6,17 +7,18 @@ namespace gaseous_server public class TimedHostedService : IHostedService, IDisposable { private int executionCount = 0; - private readonly ILogger _logger; + //private readonly ILogger _logger; private Timer _timer; - public TimedHostedService(ILogger logger) - { - _logger = logger; - } + //public TimedHostedService(ILogger logger) + //{ + // _logger = logger; + //} public Task StartAsync(CancellationToken stoppingToken) { - _logger.LogInformation("Timed Hosted Service running."); + //_logger.LogInformation("Timed Hosted Service running."); + Logging.Log(Logging.LogType.Debug, "Background", "Starting background task monitor"); _timer = new Timer(DoWork, null, TimeSpan.Zero, TimeSpan.FromSeconds(5)); @@ -28,13 +30,20 @@ namespace gaseous_server { var count = Interlocked.Increment(ref executionCount); - _logger.LogInformation( - "Timed Hosted Service is working. Count: {Count}", count); + //_logger.LogInformation( + // "Timed Hosted Service is working. Count: {Count}", count); + + foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) { + if (DateTime.UtcNow > qi.NextRunTime || qi.Force == true) { + qi.Execute(); + } + } } public Task StopAsync(CancellationToken stoppingToken) { - _logger.LogInformation("Timed Hosted Service is stopping."); + //_logger.LogInformation("Timed Hosted Service is stopping."); + Logging.Log(Logging.LogType.Debug, "Background", "Stopping background task monitor"); _timer?.Change(Timeout.Infinite, 0); diff --git a/gaseous-server/gaseous-server.csproj b/gaseous-server/gaseous-server.csproj index 35f78b0..319ca1a 100644 --- a/gaseous-server/gaseous-server.csproj +++ b/gaseous-server/gaseous-server.csproj @@ -10,8 +10,9 @@ - + + @@ -21,6 +22,9 @@ + + + diff --git a/gaseous-tools/Config.cs b/gaseous-tools/Config.cs index 948b2ac..516b747 100644 --- a/gaseous-tools/Config.cs +++ b/gaseous-tools/Config.cs @@ -1,5 +1,6 @@ using System; using System.Data; +using Google.Protobuf.WellKnownTypes; using Newtonsoft.Json; namespace gaseous_tools @@ -40,6 +41,35 @@ namespace gaseous_tools } } + public static string LogPath + { + get + { + string logPath = Path.Combine(ConfigurationPath, "Logs"); + if (!Directory.Exists(logPath)) { + Directory.CreateDirectory(logPath); + } + return logPath; + } + } + + public static string LogFilePath + { + get + { + string logPathName = Path.Combine(LogPath, "Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + ".txt"); + return logPathName; + } + } + + public static ConfigFile.Logging LoggingConfiguration + { + get + { + return _config.LoggingConfiguration; + } + } + static Config() { if (_config == null) @@ -61,8 +91,7 @@ namespace gaseous_tools // no config file! // use defaults and save _config = new ConfigFile(); - string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, Newtonsoft.Json.Formatting.Indented); - File.WriteAllText(ConfigurationFilePath, configRaw); + UpdateConfig(); } } @@ -73,7 +102,14 @@ namespace gaseous_tools public static void UpdateConfig() { // save any updates to the configuration - string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, Newtonsoft.Json.Formatting.Indented); + Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings + { + NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore, + Formatting = Newtonsoft.Json.Formatting.Indented + }; + serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter()); + string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, serializerSettings); + if (File.Exists(ConfigurationFilePath_Backup)) { File.Delete(ConfigurationFilePath_Backup); @@ -85,7 +121,7 @@ namespace gaseous_tools File.WriteAllText(ConfigurationFilePath, configRaw); } - private static string ReadSetting(string SettingName, string DefaultValue) + public static string ReadSetting(string SettingName, string DefaultValue) { Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); string sql = "SELECT * FROM settings WHERE setting = @settingname"; @@ -93,19 +129,28 @@ namespace gaseous_tools dbDict.Add("settingname", SettingName); dbDict.Add("value", DefaultValue); - DataTable dbResponse = db.ExecuteCMD(sql, dbDict); - if (dbResponse.Rows.Count == 0) + try { - // no value with that name stored - respond with the default value - return DefaultValue; + Logging.Log(Logging.LogType.Debug, "Database", "Reading setting '" + SettingName + "'"); + DataTable dbResponse = db.ExecuteCMD(sql, dbDict); + if (dbResponse.Rows.Count == 0) + { + // no value with that name stored - respond with the default value + return DefaultValue; + } + else + { + return (string)dbResponse.Rows[0][0]; + } } - else + catch (Exception ex) { - return (string)dbResponse.Rows[0][0]; + Logging.Log(Logging.LogType.Critical, "Database", "Failed reading setting " + SettingName, ex); + throw; } } - private static void SetSetting(string SettingName, string Value) + public static void SetSetting(string SettingName, string Value) { Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); string sql = "REPLACE INTO settings (setting, value) VALUES (@settingname, @value)"; @@ -113,7 +158,16 @@ namespace gaseous_tools dbDict.Add("settingname", SettingName); dbDict.Add("value", Value); - db.ExecuteCMD(sql, dbDict); + Logging.Log(Logging.LogType.Debug, "Database", "Storing setting '" + SettingName + "' to value: '" + Value + "'"); + try + { + db.ExecuteCMD(sql, dbDict); + } + catch (Exception ex) + { + Logging.Log(Logging.LogType.Critical, "Database", "Failed storing setting" + SettingName, ex); + throw; + } } public class ConfigFile @@ -123,6 +177,8 @@ namespace gaseous_tools [JsonIgnore] public Library LibraryConfiguration = new Library(); + public Logging LoggingConfiguration = new Logging(); + public class Database { public string HostName = "localhost"; @@ -180,6 +236,19 @@ namespace gaseous_tools } } } + + public class Logging + { + public bool DebugLogging = false; + + public LoggingFormat LogFormat = Logging.LoggingFormat.Json; + + public enum LoggingFormat + { + Json, + Text + } + } } } } diff --git a/gaseous-tools/Database.cs b/gaseous-tools/Database.cs index e11afc0..608c426 100644 --- a/gaseous-tools/Database.cs +++ b/gaseous-tools/Database.cs @@ -64,6 +64,7 @@ namespace gaseous_tools // check if the database exists first - first run must have permissions to create a database string sql = "CREATE DATABASE IF NOT EXISTS `" + Config.DatabaseConfiguration.DatabaseName + "`;"; Dictionary dbDict = new Dictionary(); + Logging.Log(Logging.LogType.Information, "Database", "Creating database if it doesn't exist"); ExecuteCMD(sql, dbDict, 30, "server=" + Config.DatabaseConfiguration.HostName + ";port=" + Config.DatabaseConfiguration.Port + ";userid=" + Config.DatabaseConfiguration.UserName + ";password=" + Config.DatabaseConfiguration.Password); // check if schema version table is in place - if not, create the schema version table @@ -71,8 +72,9 @@ namespace gaseous_tools DataTable SchemaVersionPresent = ExecuteCMD(sql, dbDict); if (SchemaVersionPresent.Rows.Count == 0) { - // no schema table present - create it - sql = "CREATE TABLE `schema_version` (`schema_version` INT NOT NULL, PRIMARY KEY (`schema_version`)); INSERT INTO `schema_version` (`schema_version`) VALUES (0);"; + // no schema table present - create it + Logging.Log(Logging.LogType.Information, "Database", "Schema version table doesn't exist. Creating it."); + sql = "CREATE TABLE `schema_version` (`schema_version` INT NOT NULL, PRIMARY KEY (`schema_version`)); INSERT INTO `schema_version` (`schema_version`) VALUES (0);"; ExecuteCMD(sql, dbDict); } @@ -87,7 +89,7 @@ namespace gaseous_tools using (Stream stream = assembly.GetManifestResourceStream(resourceName)) using (StreamReader reader = new StreamReader(stream)) { - dbScript = reader.ReadToEnd(); + dbScript = reader.ReadToEnd(); // apply script sql = "SELECT schema_version FROM schema_version;"; @@ -95,16 +97,19 @@ namespace gaseous_tools DataTable SchemaVersion = ExecuteCMD(sql, dbDict); if (SchemaVersion.Rows.Count == 0) { - // something is broken here... where's the table? - throw new Exception("schema_version table is missing!"); + // something is broken here... where's the table? + Logging.Log(Logging.LogType.Critical, "Database", "Schema table missing! This shouldn't happen!"); + throw new Exception("schema_version table is missing!"); } else { int SchemaVer = (int)SchemaVersion.Rows[0][0]; - if (SchemaVer < i) + Logging.Log(Logging.LogType.Information, "Database", "Schema version is " + SchemaVer); + if (SchemaVer < i) { - // apply schema! - ExecuteCMD(dbScript, dbDict); + // apply schema! + Logging.Log(Logging.LogType.Information, "Database", "Schema update available - applying"); + ExecuteCMD(dbScript, dbDict); sql = "UPDATE schema_version SET schema_version=@schemaver"; dbDict = new Dictionary(); @@ -115,7 +120,8 @@ namespace gaseous_tools } } } - break; + Logging.Log(Logging.LogType.Information, "Database", "Database setup complete"); + break; } } @@ -161,6 +167,7 @@ namespace gaseous_tools { DataTable RetTable = new DataTable(); + Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database"); MySqlConnection conn = new MySqlConnection(DBConn); conn.Open(); @@ -178,12 +185,20 @@ namespace gaseous_tools try { - RetTable.Load(cmd.ExecuteReader()); + Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'"); + if (Parameters.Count > 0) + { + string dictValues = string.Join(";", Parameters.Select(x => string.Join("=", x.Key, x.Value))); + Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues); + } + RetTable.Load(cmd.ExecuteReader()); } catch (Exception ex) { + Logging.Log(Logging.LogType.Critical, "Database", "Error while executing '" + SQL + "'", ex); Trace.WriteLine("Error executing " + SQL); Trace.WriteLine("Full exception: " + ex.ToString()); } + Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection"); conn.Close(); return RetTable; diff --git a/gaseous-tools/Logging.cs b/gaseous-tools/Logging.cs new file mode 100644 index 0000000..0243cbf --- /dev/null +++ b/gaseous-tools/Logging.cs @@ -0,0 +1,102 @@ +using System; +namespace gaseous_tools +{ + public class Logging + { + static public void Log(LogType EventType, string Section, string Message, Exception? ExceptionValue = null) + { + LogItem logItem = new LogItem + { + EventTime = DateTime.UtcNow, + EventType = EventType, + Section = Section, + Message = Message, + ExceptionValue = ExceptionValue + }; + + bool AllowWrite = false; + if (EventType == LogType.Debug) + { + if (Config.LoggingConfiguration.DebugLogging == true) + { + AllowWrite = true; + } + } + else + { + AllowWrite = true; + } + + if (AllowWrite == true) + { + // console output + string TraceOutput = logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Section + ": " + logItem.Message; + if (logItem.ExceptionValue != null) + { + TraceOutput += Environment.NewLine + logItem.ExceptionValue.ToString(); + } + Console.WriteLine(TraceOutput); + + StreamWriter LogFile = File.AppendText(Config.LogFilePath); + switch (Config.LoggingConfiguration.LogFormat) + { + case Config.ConfigFile.Logging.LoggingFormat.Text: + LogFile.WriteLine(TraceOutput); + break; + + case Config.ConfigFile.Logging.LoggingFormat.Json: + Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings + { + NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore, + Formatting = Newtonsoft.Json.Formatting.Indented + }; + serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter()); + string JsonOutput = Newtonsoft.Json.JsonConvert.SerializeObject(logItem, serializerSettings); + LogFile.WriteLine(JsonOutput); + break; + } + LogFile.Close(); + } + } + + public enum LogType + { + Information = 0, + Debug = 1, + Warning = 2, + Critical = 3 + } + + public class LogItem + { + public DateTime EventTime { get; set; } + public LogType EventType { get; set; } + private string _Section = ""; + public string Section + { + get + { + return _Section; + } + set + { + _Section = value; + } + } + private string _Message = ""; + public string Message + { + get + { + return _Message; + } + set + { + _Message = value; + } + } + public Exception? ExceptionValue { get; set; } + } + } +} +