sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Logging now writes to the database (#110)

Browse Source
This commit is contained in:
Michael Green
2023-09-18 16:54:37 +10:00
committed by GitHub
parent 031edd7088
commit 67447d49b5
9 changed files with 131 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
gaseous-server/Models/PlatformMapping.cs
View File

@@ -120,7 +120,9 @@ namespace gaseous_server.Models
} }
else else
{ {
throw new Exception(""); Exception exception = new Exception("Platform Map Id " + Id + " does not exist.");
Logging.Log(Logging.LogType.Critical, "Platform Map", "Platform Map Id " + Id + " does not exist.", exception);
throw exception;
} }
} }
@@ -162,11 +164,14 @@ namespace gaseous_server.Models
{ {
foreach (string alternateName in item.AlternateNames) foreach (string alternateName in item.AlternateNames)
{ {
sql = "INSERT INTO PlatformMap_AlternateNames (Id, Name) VALUES (@Id, @Name);"; if (alternateName != null)
dbDict.Clear(); {
dbDict.Add("Id", item.IGDBId); sql = "INSERT INTO PlatformMap_AlternateNames (Id, Name) VALUES (@Id, @Name);";
dbDict.Add("Name", alternateName); dbDict.Clear();
db.ExecuteCMD(sql, dbDict); dbDict.Add("Id", item.IGDBId);
dbDict.Add("Name", alternateName);
db.ExecuteCMD(sql, dbDict);
}
} }
} }

20
gaseous-server/ProcessQueue.cs
View File

@@ -68,49 +68,49 @@ namespace gaseous_server
_LastResult = ""; _LastResult = "";
_LastError = null; _LastError = null;
Logging.Log(Logging.LogType.Information, "Timered Event", "Executing " + _ItemType); Logging.Log(Logging.LogType.Debug, "Timered Event", "Executing " + _ItemType);
try try
{ {
switch (_ItemType) switch (_ItemType)
{ {
case QueueItemType.SignatureIngestor: case QueueItemType.SignatureIngestor:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Signature Ingestor"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Signature Ingestor");
SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor(); SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor();
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing TOSEC files"); Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing TOSEC files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "TOSEC"), gaseous_signature_parser.parser.SignatureParser.TOSEC); tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "TOSEC"), gaseous_signature_parser.parser.SignatureParser.TOSEC);
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing MAME Arcade files"); Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME Arcade files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME Arcade"), gaseous_signature_parser.parser.SignatureParser.MAMEArcade); tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME Arcade"), gaseous_signature_parser.parser.SignatureParser.MAMEArcade);
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing MAME MESS files"); Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME MESS files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess); tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess);
break; break;
case QueueItemType.TitleIngestor: case QueueItemType.TitleIngestor:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Title Ingestor"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor");
Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory); Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory);
break; break;
case QueueItemType.MetadataRefresh: case QueueItemType.MetadataRefresh:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Metadata Refresher"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher");
Classes.MetadataManagement.RefreshMetadata(true); Classes.MetadataManagement.RefreshMetadata(true);
break; break;
case QueueItemType.OrganiseLibrary: case QueueItemType.OrganiseLibrary:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Library Organiser"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Organiser");
Classes.ImportGame.OrganiseLibrary(); Classes.ImportGame.OrganiseLibrary();
break; break;
case QueueItemType.LibraryScan: case QueueItemType.LibraryScan:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Library Scanner"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner");
Classes.ImportGame.LibraryScan(); Classes.ImportGame.LibraryScan();
break; break;
case QueueItemType.CollectionCompiler: case QueueItemType.CollectionCompiler:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Collection Compiler"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
Classes.Collections.CompileCollections(); Classes.Collections.CompileCollections();
break; break;
} }

2
gaseous-server/Support/PlatformMap.json
View File

File diff suppressed because one or more lines are too long

25
gaseous-server/wwwroot/pages/settings/logs.html
View File

@@ -3,7 +3,7 @@
</div> </div>
<a href="#" class="romlink" onclick="loadLogs();" style="float: right;"><img src="/images/refresh.svg" alt="Refresh" title="Refresh" class="banner_button_image" /></a> <a href="#" class="romlink" onclick="loadLogs();" style="float: right;"><img src="/images/refresh.svg" alt="Refresh" title="Refresh" class="banner_button_image" /></a>
<table id="settings_events_table" style="width: 100%;" cellspacing="0"> <table id="settings_events_table" style="width: 960px; max-width: 960px;" cellspacing="0">
</table> </table>
@@ -21,6 +21,7 @@
[ [
['Event Time', 'logs_table_cell_150px'], ['Event Time', 'logs_table_cell_150px'],
['Severity', 'logs_table_cell_150px'], ['Severity', 'logs_table_cell_150px'],
'Process',
'Message' 'Message'
], ],
'', '',
@@ -29,18 +30,26 @@
); );
for (var i = 0; i < result.length; i++) { for (var i = 0; i < result.length; i++) {
var exceptionString = '';
if (result[i].exceptionValue) {
exceptionString = "<h3>Exception</h3><pre class='logs_table_exception'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br /> ") + "</pre>";
}
var newRow = [ var newRow = [
moment(result[i].eventTime).fromNow(), moment(result[i].eventTime).fromNow(),
result[i].eventType, result[i].eventType,
result[i].process + " - " + result[i].message + exceptionString result[i].process,
result[i].message
]; ];
newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell logs_table_cell')); newTable.appendChild(createTableRow(false, newRow, 'romrow logs_table_row_' + result[i].eventType, 'romcell logs_table_cell'));
if (result[i].exceptionValue) {
var exceptionString = "<h3>Exception</h3><pre class='logs_table_exception' style='width: 795px; word-wrap: break-word; overflow-wrap: break-word; overflow-y: scroll;'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br /> ") + "</pre>";
var exRow = document.createElement('tr');
var leadCell = document.createElement('td');
exRow.appendChild(leadCell);
var exCell = document.createElement('td');
exCell.colSpan = '3';
exCell.innerHTML = exceptionString;
exRow.appendChild(exCell);
newTable.appendChild(exRow);
}
} }
} }
); );

16
gaseous-server/wwwroot/styles/style.css
View File

@@ -893,6 +893,22 @@ button:disabled {
vertical-align: top; vertical-align: top;
} }
.logs_table_row_Information:hover {
background: rgba(42, 41, 150, 0.3);
}
.logs_table_row_Warning:hover {
background: rgba(139, 150, 41, 0.3);
}
.logs_table_row_Critical:hover {
background: rgba(150, 41, 41, 0.3);
}
.logs_table_row_Debug:hover {
background: rgba(150, 41, 135, 0.3);
}
.logs_table_exception { .logs_table_exception {
margin-right: 10px; margin-right: 10px;
padding: 5px; padding: 5px;

5
gaseous-tools/Config.cs
View File

@@ -82,7 +82,7 @@ namespace gaseous_tools
{ {
get get
{ {
string logFileExtension = "json"; string logFileExtension = "txt";
string logPathName = Path.Combine(LogPath, "Server Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + "." + logFileExtension); string logPathName = Path.Combine(LogPath, "Server Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + "." + logFileExtension);
return logPathName; return logPathName;
@@ -468,7 +468,8 @@ namespace gaseous_tools
{ {
public bool DebugLogging = false; public bool DebugLogging = false;
public int LogRetention = 30; // log retention in days
public int LogRetention = 7;
public enum LoggingFormat public enum LoggingFormat
{ {

8
gaseous-tools/Database.cs
View File

@@ -236,7 +236,7 @@ namespace gaseous_tools
{ {
DataTable RetTable = new DataTable(); DataTable RetTable = new DataTable();
Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database"); Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database", null, true);
MySqlConnection conn = new MySqlConnection(DBConn); MySqlConnection conn = new MySqlConnection(DBConn);
conn.Open(); conn.Open();
@@ -254,11 +254,11 @@ namespace gaseous_tools
try try
{ {
Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'"); Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'", null, true);
if (Parameters.Count > 0) if (Parameters.Count > 0)
{ {
string dictValues = string.Join(";", Parameters.Select(x => string.Join("=", x.Key, x.Value))); string dictValues = string.Join(";", Parameters.Select(x => string.Join("=", x.Key, x.Value)));
Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues); Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues, null, true);
} }
RetTable.Load(cmd.ExecuteReader()); RetTable.Load(cmd.ExecuteReader());
} catch (Exception ex) { } catch (Exception ex) {
@@ -267,7 +267,7 @@ namespace gaseous_tools
Trace.WriteLine("Full exception: " + ex.ToString()); Trace.WriteLine("Full exception: " + ex.ToString());
} }
Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection"); Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection", null, true);
conn.Close(); conn.Close();
return RetTable; return RetTable;

12
gaseous-tools/Database/MySQL/gaseous-1002.sql
View File

@@ -44,3 +44,15 @@ CREATE TABLE `PlatformMap_Bios` (
`Description` LONGTEXT NOT NULL, `Description` LONGTEXT NOT NULL,
`Hash` VARCHAR(45) NOT NULL, `Hash` VARCHAR(45) NOT NULL,
PRIMARY KEY (`Id`, `Filename`, `Hash`)); PRIMARY KEY (`Id`, `Filename`, `Hash`));
CREATE TABLE `ServerLogs` (
`Id` BIGINT NOT NULL AUTO_INCREMENT,
`EventTime` DATETIME NOT NULL,
`EventType` INT NOT NULL,
`Process` VARCHAR(100) NOT NULL,
`Message` LONGTEXT NOT NULL,
`Exception` LONGTEXT NULL,
PRIMARY KEY (`Id`));
ALTER TABLE `PlatformVersion`
CHANGE COLUMN `PlatformLogo` `PlatformLogo` BIGINT NULL DEFAULT NULL ;

112
gaseous-tools/Logging.cs
View File

@@ -1,4 +1,5 @@
using System; using System;
using System.Data;
using System.Diagnostics; using System.Diagnostics;
using System.Reflection; using System.Reflection;
using System.Reflection.Metadata.Ecma335; using System.Reflection.Metadata.Ecma335;
@@ -7,12 +8,7 @@ namespace gaseous_tools
{ {
public class Logging public class Logging
{ {
// when was the last clean static public void Log(LogType EventType, string ServerProcess, string Message, Exception? ExceptionValue = null, bool LogToDiskOnly = false)
static DateTime LastRetentionClean = DateTime.UtcNow;
// how often to clean in hours
const int RetentionCleanInterval = 1;
static public void Log(LogType EventType, string ServerProcess, string Message, Exception? ExceptionValue = null)
{ {
LogItem logItem = new LogItem LogItem logItem = new LogItem
{ {
@@ -20,7 +16,7 @@ namespace gaseous_tools
EventType = EventType, EventType = EventType,
Process = ServerProcess, Process = ServerProcess,
Message = Message, Message = Message,
ExceptionValue = ExceptionValue ExceptionValue = Common.ReturnValueIfNull(ExceptionValue, "").ToString()
}; };
bool AllowWrite = false; bool AllowWrite = false;
@@ -65,67 +61,73 @@ namespace gaseous_tools
Console.WriteLine(TraceOutput); Console.WriteLine(TraceOutput);
Console.ResetColor(); Console.ResetColor();
Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings if (LogToDiskOnly == false)
{ {
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore, Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
Formatting = Newtonsoft.Json.Formatting.None string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception) VALUES (@EventTime, @EventType, @Process, @Message, @Exception);";
}; Dictionary<string, object> dbDict = new Dictionary<string, object>();
serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter()); dbDict.Add("EventRententionDate", DateTime.UtcNow.AddDays(Config.LoggingConfiguration.LogRetention * -1));
dbDict.Add("EventTime", logItem.EventTime);
dbDict.Add("EventType", logItem.EventType);
dbDict.Add("Process", logItem.Process);
dbDict.Add("Message", logItem.Message);
dbDict.Add("Exception", Common.ReturnValueIfNull(logItem.ExceptionValue, "").ToString());
// write log file try
string JsonOutput = Newtonsoft.Json.JsonConvert.SerializeObject(logItem, serializerSettings); {
File.AppendAllText(Config.LogFilePath, JsonOutput); db.ExecuteCMD(sql, dbDict);
} }
catch (Exception ex)
// quick clean before we go {
if (LastRetentionClean.AddHours(RetentionCleanInterval) < DateTime.UtcNow) LogToDisk(logItem, TraceOutput, ex);
{ }
LogCleanup(); }
else
{
LogToDisk(logItem, TraceOutput, null);
}
} }
} }
static public List<LogItem> GetLogs() { static void LogToDisk(LogItem logItem, string TraceOutput, Exception? exception)
string logData = File.ReadAllText(Config.LogFilePath); {
if (exception != null)
{
// dump the error
File.AppendAllText(Config.LogFilePath, logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Process + ": " + logItem.Message + Environment.NewLine + exception.ToString());
// something went wrong writing to the db
File.AppendAllText(Config.LogFilePath, logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": The following event was unable to be written to the log database:");
}
File.AppendAllText(Config.LogFilePath, TraceOutput);
}
static public List<LogItem> GetLogs()
{
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT * FROM ServerLogs ORDER BY Id DESC";
DataTable dataTable = db.ExecuteCMD(sql);
List<LogItem> logs = new List<LogItem>(); List<LogItem> logs = new List<LogItem>();
if (File.Exists(Config.LogFilePath)) foreach (DataRow row in dataTable.Rows)
{ {
StreamReader sr = new StreamReader(Config.LogFilePath); LogItem log = new LogItem
while (!sr.EndOfStream)
{ {
LogItem logItem = Newtonsoft.Json.JsonConvert.DeserializeObject<LogItem>(sr.ReadLine()); EventTime = DateTime.Parse(((DateTime)row["EventTime"]).ToString("yyyy-MM-ddThh:mm:ss") + 'Z'),
logs.Add(logItem); EventType = (LogType)row["EventType"],
} Process = (string)row["Process"],
logs.Reverse(); Message = (string)row["Message"],
ExceptionValue = (string)row["Exception"]
};
logs.Add(log);
} }
return logs; return logs;
} }
static public void LogCleanup()
{
Log(LogType.Information, "Log Cleanup", "Purging log files older than " + Config.LoggingConfiguration.LogRetention + " days");
LastRetentionClean = DateTime.UtcNow;
string[] files = Directory.GetFiles(Config.LogPath, "Server Log *.json");
foreach (string file in files)
{
FileInfo fi = new FileInfo(file);
if (fi.LastAccessTime.AddDays(Config.LoggingConfiguration.LogRetention) < DateTime.Now)
{
try
{
fi.Delete();
}
catch
{
Log(LogType.Warning, "Log Cleanup", "Failed purging log " + fi.FullName);
}
}
}
}
public enum LogType public enum LogType
{ {
Information = 0, Information = 0,
@@ -151,7 +153,7 @@ namespace gaseous_tools
_Message = value; _Message = value;
} }
} }
public Exception? ExceptionValue { get; set; } public string? ExceptionValue { get; set; }
} }
} }
} }