sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: scaffolding for background tasks complete

Browse Source
This commit is contained in:
Michael Green
2023-03-17 23:08:46 +11:00
parent 55735599f8
commit 5260738a5b
9 changed files with 397 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
gaseous-server/Controllers/BackgroundTasksController.cs Normal file
View File

@@ -0,0 +1,41 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
namespace gaseous_server.Controllers
{
[ApiController]
[Route("api/v1/[controller]")]
public class BackgroundTasksController : Controller
{
[HttpGet]
[ProducesResponseType(StatusCodes.Status200OK)]
public List<ProcessQueue.QueueItem> GetQueue()
{
return ProcessQueue.QueueItems;
}
[HttpGet]
[Route("{TaskType}")]
[ProducesResponseType(StatusCodes.Status200OK)]
[ProducesResponseType(StatusCodes.Status404NotFound)]
public ActionResult<ProcessQueue.QueueItem> ForceRun(ProcessQueue.QueueItemType TaskType, Boolean ForceRun)
{
foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
{
if (TaskType == qi.ItemType)
{
if (ForceRun == true)
{
qi.ForceExecute();
}
return qi;
}
}
return NotFound();
}
}
}

3
gaseous-server/Controllers/SignaturesController.cs
View File

@@ -20,13 +20,14 @@ namespace gaseous_server.Controllers
/// </summary>
/// <returns>Number of sources, publishers, games, and rom signatures in the database</returns>
[HttpGet]
[ProducesResponseType(StatusCodes.Status200OK)]
public Models.Signatures_Status Status()
{
return new Models.Signatures_Status();
}
[HttpGet]
[Route("api/[controller]/[action]")]
[ProducesResponseType(StatusCodes.Status200OK)]
public List<Models.Signatures_Games> GetSignature(string md5 = "", string sha1 = "")
{
if (md5.Length > 0)

107
gaseous-server/ProcessQueue.cs Normal file
View File

@@ -0,0 +1,107 @@
using System;
using gaseous_tools;
namespace gaseous_server
{
public static class ProcessQueue
{
public static List<QueueItem> QueueItems = new List<QueueItem>();
public class QueueItem
{
public QueueItem(QueueItemType ItemType, int ExecutionInterval)
{
_ItemType = ItemType;
_ItemState = QueueItemState.NeverStarted;
_LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval);
_Interval = ExecutionInterval;
}
private QueueItemType _ItemType = QueueItemType.NotConfigured;
private QueueItemState _ItemState = QueueItemState.NeverStarted;
private DateTime _LastRunTime = DateTime.UtcNow;
private DateTime _LastFinishTime = DateTime.UtcNow;
private int _Interval = 0;
private string _LastResult = "";
private Exception? _LastError = null;
private bool _ForceExecute = false;
public QueueItemType ItemType => _ItemType;
public QueueItemState ItemState => _ItemState;
public DateTime LastRunTime => _LastRunTime;
public DateTime LastFinishTime => _LastFinishTime;
public DateTime NextRunTime {
get
{
return LastRunTime.AddMinutes(Interval);
}
}
public int Interval => _Interval;
public string LastResult => _LastResult;
public Exception? LastError => _LastError;
public bool Force => _ForceExecute;
public void Execute()
{
if (_ItemState != QueueItemState.Disabled)
{
if ((DateTime.UtcNow > NextRunTime || _ForceExecute == true) && _ItemState != QueueItemState.Running)
{
// we can run - do some setup before we start processing
_LastRunTime = DateTime.UtcNow;
_ItemState = QueueItemState.Running;
_LastResult = "";
_LastError = null;
_ForceExecute = false;
Logging.Log(Logging.LogType.Information, "Timered Event", "Executing " + _ItemType);
try
{
switch (_ItemType)
{
case QueueItemType.SignatureIngestor:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Signature Ingestor");
break;
case QueueItemType.TitleIngestor:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Title Ingestor");
break;
}
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Warning, "Timered Event", "An error occurred", ex);
_LastResult = "";
_LastError = ex;
}
_ItemState = QueueItemState.Stopped;
_LastFinishTime = DateTime.UtcNow;
}
}
}
public void ForceExecute()
{
_ForceExecute = true;
}
}
public enum QueueItemType
{
NotConfigured,
SignatureIngestor,
TitleIngestor
}
public enum QueueItemState
{
NeverStarted,
Running,
Stopped,
Disabled
}
}
}

18
gaseous-server/Program.cs
View File

@@ -2,20 +2,31 @@
using gaseous_server;
using gaseous_tools;
Logging.Log(Logging.LogType.Information, "Startup", "Starting Gaseous Server");
// set up db
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
db.InitDB();
// set initial values
Guid APIKey = Guid.NewGuid();
if (Config.ReadSetting("API Key", "Test API Key") == "Test API Key")
{
// it's a new api key save it
Logging.Log(Logging.LogType.Information, "Startup", "Setting initial API key");
Config.SetSetting("API Key", APIKey.ToString());
}
// set up server
var builder = WebApplication.CreateBuilder(args);
// Add services to the container.
builder.Services.AddControllers().AddJsonOptions(x =>
{
// serialize enums as strings in api responses (e.g. Role)
x.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter());
});
// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();
@@ -36,6 +47,9 @@ app.UseAuthorization();
app.MapControllers();
// add background tasks
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60));
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.TitleIngestor, 1));
// start the app
app.Run();

27
gaseous-server/Timer.cs
View File

@@ -1,4 +1,5 @@
using System;
using gaseous_tools;
namespace gaseous_server
{
@@ -6,17 +7,18 @@ namespace gaseous_server
public class TimedHostedService : IHostedService, IDisposable
{
private int executionCount = 0;
private readonly ILogger<TimedHostedService> _logger;
//private readonly ILogger<TimedHostedService> _logger;
private Timer _timer;
public TimedHostedService(ILogger<TimedHostedService> logger)
{
_logger = logger;
}
//public TimedHostedService(ILogger<TimedHostedService> logger)
//{
// _logger = logger;
//}
public Task StartAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Timed Hosted Service running.");
//_logger.LogInformation("Timed Hosted Service running.");
Logging.Log(Logging.LogType.Debug, "Background", "Starting background task monitor");
_timer = new Timer(DoWork, null, TimeSpan.Zero,
TimeSpan.FromSeconds(5));
@@ -28,13 +30,20 @@ namespace gaseous_server
{
var count = Interlocked.Increment(ref executionCount);
_logger.LogInformation(
"Timed Hosted Service is working. Count: {Count}", count);
//_logger.LogInformation(
// "Timed Hosted Service is working. Count: {Count}", count);
foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) {
if (DateTime.UtcNow > qi.NextRunTime || qi.Force == true) {
qi.Execute();
}
}
}
public Task StopAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Timed Hosted Service is stopping.");
//_logger.LogInformation("Timed Hosted Service is stopping.");
Logging.Log(Logging.LogType.Debug, "Background", "Stopping background task monitor");
_timer?.Change(Timeout.Infinite, 0);

6
gaseous-server/gaseous-server.csproj
View File

@@ -10,8 +10,9 @@
<PropertyGroup Condition=" '$(RunConfiguration)' == 'https' " />
<PropertyGroup Condition=" '$(RunConfiguration)' == 'http' " />
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.3" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.4" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="7.0.5" />
</ItemGroup>
<ItemGroup>
@@ -21,6 +22,9 @@
<ProjectReference Include="..\gaseous-romsignatureobject\gaseous-romsignatureobject.csproj">
<GlobalPropertiesToRemove></GlobalPropertiesToRemove>
</ProjectReference>
<ProjectReference Include="..\gaseous-signature-ingestor\gaseous-signature-ingestor.csproj">
<GlobalPropertiesToRemove></GlobalPropertiesToRemove>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Remove="Controllers\" />

79
gaseous-tools/Config.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Data;
using Google.Protobuf.WellKnownTypes;
using Newtonsoft.Json;
namespace gaseous_tools
@@ -40,6 +41,35 @@ namespace gaseous_tools
}
}
public static string LogPath
{
get
{
string logPath = Path.Combine(ConfigurationPath, "Logs");
if (!Directory.Exists(logPath)) {
Directory.CreateDirectory(logPath);
}
return logPath;
}
}
public static string LogFilePath
{
get
{
string logPathName = Path.Combine(LogPath, "Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + ".txt");
return logPathName;
}
}
public static ConfigFile.Logging LoggingConfiguration
{
get
{
return _config.LoggingConfiguration;
}
}
static Config()
{
if (_config == null)
@@ -61,8 +91,7 @@ namespace gaseous_tools
// no config file!
// use defaults and save
_config = new ConfigFile();
string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, Newtonsoft.Json.Formatting.Indented);
File.WriteAllText(ConfigurationFilePath, configRaw);
UpdateConfig();
}
}
@@ -73,7 +102,14 @@ namespace gaseous_tools
public static void UpdateConfig()
{
// save any updates to the configuration
string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, Newtonsoft.Json.Formatting.Indented);
Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings
{
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
Formatting = Newtonsoft.Json.Formatting.Indented
};
serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
string configRaw = Newtonsoft.Json.JsonConvert.SerializeObject(_config, serializerSettings);
if (File.Exists(ConfigurationFilePath_Backup))
{
File.Delete(ConfigurationFilePath_Backup);
@@ -85,7 +121,7 @@ namespace gaseous_tools
File.WriteAllText(ConfigurationFilePath, configRaw);
}
private static string ReadSetting(string SettingName, string DefaultValue)
public static string ReadSetting(string SettingName, string DefaultValue)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT * FROM settings WHERE setting = @settingname";
@@ -93,6 +129,9 @@ namespace gaseous_tools
dbDict.Add("settingname", SettingName);
dbDict.Add("value", DefaultValue);
try
{
Logging.Log(Logging.LogType.Debug, "Database", "Reading setting '" + SettingName + "'");
DataTable dbResponse = db.ExecuteCMD(sql, dbDict);
if (dbResponse.Rows.Count == 0)
{
@@ -104,8 +143,14 @@ namespace gaseous_tools
return (string)dbResponse.Rows[0][0];
}
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Critical, "Database", "Failed reading setting " + SettingName, ex);
throw;
}
}
private static void SetSetting(string SettingName, string Value)
public static void SetSetting(string SettingName, string Value)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "REPLACE INTO settings (setting, value) VALUES (@settingname, @value)";
@@ -113,8 +158,17 @@ namespace gaseous_tools
dbDict.Add("settingname", SettingName);
dbDict.Add("value", Value);
Logging.Log(Logging.LogType.Debug, "Database", "Storing setting '" + SettingName + "' to value: '" + Value + "'");
try
{
db.ExecuteCMD(sql, dbDict);
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Critical, "Database", "Failed storing setting" + SettingName, ex);
throw;
}
}
public class ConfigFile
{
@@ -123,6 +177,8 @@ namespace gaseous_tools
[JsonIgnore]
public Library LibraryConfiguration = new Library();
public Logging LoggingConfiguration = new Logging();
public class Database
{
public string HostName = "localhost";
@@ -180,6 +236,19 @@ namespace gaseous_tools
}
}
}
public class Logging
{
public bool DebugLogging = false;
public LoggingFormat LogFormat = Logging.LoggingFormat.Json;
public enum LoggingFormat
{
Json,
Text
}
}
}
}
}

15
gaseous-tools/Database.cs
View File

@@ -64,6 +64,7 @@ namespace gaseous_tools
// check if the database exists first - first run must have permissions to create a database
string sql = "CREATE DATABASE IF NOT EXISTS `" + Config.DatabaseConfiguration.DatabaseName + "`;";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
Logging.Log(Logging.LogType.Information, "Database", "Creating database if it doesn't exist");
ExecuteCMD(sql, dbDict, 30, "server=" + Config.DatabaseConfiguration.HostName + ";port=" + Config.DatabaseConfiguration.Port + ";userid=" + Config.DatabaseConfiguration.UserName + ";password=" + Config.DatabaseConfiguration.Password);
// check if schema version table is in place - if not, create the schema version table
@@ -72,6 +73,7 @@ namespace gaseous_tools
if (SchemaVersionPresent.Rows.Count == 0)
{
// no schema table present - create it
Logging.Log(Logging.LogType.Information, "Database", "Schema version table doesn't exist. Creating it.");
sql = "CREATE TABLE `schema_version` (`schema_version` INT NOT NULL, PRIMARY KEY (`schema_version`)); INSERT INTO `schema_version` (`schema_version`) VALUES (0);";
ExecuteCMD(sql, dbDict);
}
@@ -96,14 +98,17 @@ namespace gaseous_tools
if (SchemaVersion.Rows.Count == 0)
{
// something is broken here... where's the table?
Logging.Log(Logging.LogType.Critical, "Database", "Schema table missing! This shouldn't happen!");
throw new Exception("schema_version table is missing!");
}
else
{
int SchemaVer = (int)SchemaVersion.Rows[0][0];
Logging.Log(Logging.LogType.Information, "Database", "Schema version is " + SchemaVer);
if (SchemaVer < i)
{
// apply schema!
Logging.Log(Logging.LogType.Information, "Database", "Schema update available - applying");
ExecuteCMD(dbScript, dbDict);
sql = "UPDATE schema_version SET schema_version=@schemaver";
@@ -115,6 +120,7 @@ namespace gaseous_tools
}
}
}
Logging.Log(Logging.LogType.Information, "Database", "Database setup complete");
break;
}
}
@@ -161,6 +167,7 @@ namespace gaseous_tools
{
DataTable RetTable = new DataTable();
Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database");
MySqlConnection conn = new MySqlConnection(DBConn);
conn.Open();
@@ -178,12 +185,20 @@ namespace gaseous_tools
try
{
Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'");
if (Parameters.Count > 0)
{
string dictValues = string.Join(";", Parameters.Select(x => string.Join("=", x.Key, x.Value)));
Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues);
}
RetTable.Load(cmd.ExecuteReader());
} catch (Exception ex) {
Logging.Log(Logging.LogType.Critical, "Database", "Error while executing '" + SQL + "'", ex);
Trace.WriteLine("Error executing " + SQL);
Trace.WriteLine("Full exception: " + ex.ToString());
}
Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection");
conn.Close();
return RetTable;

102
gaseous-tools/Logging.cs Normal file
View File

@@ -0,0 +1,102 @@
using System;
namespace gaseous_tools
{
public class Logging
{
static public void Log(LogType EventType, string Section, string Message, Exception? ExceptionValue = null)
{
LogItem logItem = new LogItem
{
EventTime = DateTime.UtcNow,
EventType = EventType,
Section = Section,
Message = Message,
ExceptionValue = ExceptionValue
};
bool AllowWrite = false;
if (EventType == LogType.Debug)
{
if (Config.LoggingConfiguration.DebugLogging == true)
{
AllowWrite = true;
}
}
else
{
AllowWrite = true;
}
if (AllowWrite == true)
{
// console output
string TraceOutput = logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Section + ": " + logItem.Message;
if (logItem.ExceptionValue != null)
{
TraceOutput += Environment.NewLine + logItem.ExceptionValue.ToString();
}
Console.WriteLine(TraceOutput);
StreamWriter LogFile = File.AppendText(Config.LogFilePath);
switch (Config.LoggingConfiguration.LogFormat)
{
case Config.ConfigFile.Logging.LoggingFormat.Text:
LogFile.WriteLine(TraceOutput);
break;
case Config.ConfigFile.Logging.LoggingFormat.Json:
Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings
{
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
Formatting = Newtonsoft.Json.Formatting.Indented
};
serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
string JsonOutput = Newtonsoft.Json.JsonConvert.SerializeObject(logItem, serializerSettings);
LogFile.WriteLine(JsonOutput);
break;
}
LogFile.Close();
}
}
public enum LogType
{
Information = 0,
Debug = 1,
Warning = 2,
Critical = 3
}
public class LogItem
{
public DateTime EventTime { get; set; }
public LogType EventType { get; set; }
private string _Section = "";
public string Section
{
get
{
return _Section;
}
set
{
_Section = value;
}
}
private string _Message = "";
public string Message
{
get
{
return _Message;
}
set
{
_Message = value;
}
}
public Exception? ExceptionValue { get; set; }
}
}
}