sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Logging now writes to the database (#110)

Browse Source
This commit is contained in:
Michael Green
2023-09-18 16:54:37 +10:00
committed by GitHub
parent 031edd7088
commit 67447d49b5
9 changed files with 131 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
gaseous-server/Models/PlatformMapping.cs
View File

@@ -120,7 +120,9 @@ namespace gaseous_server.Models
}
else
{
throw new Exception("");
Exception exception = new Exception("Platform Map Id " + Id + " does not exist.");
Logging.Log(Logging.LogType.Critical, "Platform Map", "Platform Map Id " + Id + " does not exist.", exception);
throw exception;
}
}
@@ -161,6 +163,8 @@ namespace gaseous_server.Models
if (item.AlternateNames != null)
{
foreach (string alternateName in item.AlternateNames)
{
if (alternateName != null)
{
sql = "INSERT INTO PlatformMap_AlternateNames (Id, Name) VALUES (@Id, @Name);";
dbDict.Clear();
@@ -169,6 +173,7 @@ namespace gaseous_server.Models
db.ExecuteCMD(sql, dbDict);
}
}
}
// insert extensions
if (item.Extensions != null)

20
gaseous-server/ProcessQueue.cs
View File

@@ -68,49 +68,49 @@ namespace gaseous_server
_LastResult = "";
_LastError = null;
Logging.Log(Logging.LogType.Information, "Timered Event", "Executing " + _ItemType);
Logging.Log(Logging.LogType.Debug, "Timered Event", "Executing " + _ItemType);
try
{
switch (_ItemType)
{
case QueueItemType.SignatureIngestor:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Signature Ingestor");
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Signature Ingestor");
SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor();
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing TOSEC files");
Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing TOSEC files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "TOSEC"), gaseous_signature_parser.parser.SignatureParser.TOSEC);
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing MAME Arcade files");
Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME Arcade files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME Arcade"), gaseous_signature_parser.parser.SignatureParser.MAMEArcade);
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing MAME MESS files");
Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME MESS files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess);
break;
case QueueItemType.TitleIngestor:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Title Ingestor");
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor");
Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory);
break;
case QueueItemType.MetadataRefresh:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Metadata Refresher");
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher");
Classes.MetadataManagement.RefreshMetadata(true);
break;
case QueueItemType.OrganiseLibrary:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Library Organiser");
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Organiser");
Classes.ImportGame.OrganiseLibrary();
break;
case QueueItemType.LibraryScan:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Library Scanner");
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner");
Classes.ImportGame.LibraryScan();
break;
case QueueItemType.CollectionCompiler:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Collection Compiler");
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
Classes.Collections.CompileCollections();
break;
}

2
gaseous-server/Support/PlatformMap.json
View File

File diff suppressed because one or more lines are too long

25
gaseous-server/wwwroot/pages/settings/logs.html
View File

@@ -3,7 +3,7 @@
</div>
<a href="#" class="romlink" onclick="loadLogs();" style="float: right;"><img src="/images/refresh.svg" alt="Refresh" title="Refresh" class="banner_button_image" /></a>
<table id="settings_events_table" style="width: 100%;" cellspacing="0">
<table id="settings_events_table" style="width: 960px; max-width: 960px;" cellspacing="0">
</table>
@@ -21,6 +21,7 @@
[
['Event Time', 'logs_table_cell_150px'],
['Severity', 'logs_table_cell_150px'],
'Process',
'Message'
],
'',
@@ -29,18 +30,26 @@
);
for (var i = 0; i < result.length; i++) {
var exceptionString = '';
if (result[i].exceptionValue) {
exceptionString = "<h3>Exception</h3><pre class='logs_table_exception'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br /> ") + "</pre>";
}
var newRow = [
moment(result[i].eventTime).fromNow(),
result[i].eventType,
result[i].process + " - " + result[i].message + exceptionString
result[i].process,
result[i].message
];
newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell logs_table_cell'));
newTable.appendChild(createTableRow(false, newRow, 'romrow logs_table_row_' + result[i].eventType, 'romcell logs_table_cell'));
if (result[i].exceptionValue) {
var exceptionString = "<h3>Exception</h3><pre class='logs_table_exception' style='width: 795px; word-wrap: break-word; overflow-wrap: break-word; overflow-y: scroll;'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br /> ") + "</pre>";
var exRow = document.createElement('tr');
var leadCell = document.createElement('td');
exRow.appendChild(leadCell);
var exCell = document.createElement('td');
exCell.colSpan = '3';
exCell.innerHTML = exceptionString;
exRow.appendChild(exCell);
newTable.appendChild(exRow);
}
}
}
);

16
gaseous-server/wwwroot/styles/style.css
View File

@@ -893,6 +893,22 @@ button:disabled {
vertical-align: top;
}
.logs_table_row_Information:hover {
background: rgba(42, 41, 150, 0.3);
}
.logs_table_row_Warning:hover {
background: rgba(139, 150, 41, 0.3);
}
.logs_table_row_Critical:hover {
background: rgba(150, 41, 41, 0.3);
}
.logs_table_row_Debug:hover {
background: rgba(150, 41, 135, 0.3);
}
.logs_table_exception {
margin-right: 10px;
padding: 5px;

5
gaseous-tools/Config.cs
View File

@@ -82,7 +82,7 @@ namespace gaseous_tools
{
get
{
string logFileExtension = "json";
string logFileExtension = "txt";
string logPathName = Path.Combine(LogPath, "Server Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + "." + logFileExtension);
return logPathName;
@@ -468,7 +468,8 @@ namespace gaseous_tools
{
public bool DebugLogging = false;
public int LogRetention = 30;
// log retention in days
public int LogRetention = 7;
public enum LoggingFormat
{

8
gaseous-tools/Database.cs
View File

@@ -236,7 +236,7 @@ namespace gaseous_tools
{
DataTable RetTable = new DataTable();
Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database");
Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database", null, true);
MySqlConnection conn = new MySqlConnection(DBConn);
conn.Open();
@@ -254,11 +254,11 @@ namespace gaseous_tools
try
{
Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'");
Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'", null, true);
if (Parameters.Count > 0)
{
string dictValues = string.Join(";", Parameters.Select(x => string.Join("=", x.Key, x.Value)));
Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues);
Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues, null, true);
}
RetTable.Load(cmd.ExecuteReader());
} catch (Exception ex) {
@@ -267,7 +267,7 @@ namespace gaseous_tools
Trace.WriteLine("Full exception: " + ex.ToString());
}
Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection");
Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection", null, true);
conn.Close();
return RetTable;

12
gaseous-tools/Database/MySQL/gaseous-1002.sql
View File

@@ -44,3 +44,15 @@ CREATE TABLE `PlatformMap_Bios` (
`Description` LONGTEXT NOT NULL,
`Hash` VARCHAR(45) NOT NULL,
PRIMARY KEY (`Id`, `Filename`, `Hash`));
CREATE TABLE `ServerLogs` (
`Id` BIGINT NOT NULL AUTO_INCREMENT,
`EventTime` DATETIME NOT NULL,
`EventType` INT NOT NULL,
`Process` VARCHAR(100) NOT NULL,
`Message` LONGTEXT NOT NULL,
`Exception` LONGTEXT NULL,
PRIMARY KEY (`Id`));
ALTER TABLE `PlatformVersion`
CHANGE COLUMN `PlatformLogo` `PlatformLogo` BIGINT NULL DEFAULT NULL ;

110
gaseous-tools/Logging.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Data;
using System.Diagnostics;
using System.Reflection;
using System.Reflection.Metadata.Ecma335;
@@ -7,12 +8,7 @@ namespace gaseous_tools
{
public class Logging
{
// when was the last clean
static DateTime LastRetentionClean = DateTime.UtcNow;
// how often to clean in hours
const int RetentionCleanInterval = 1;
static public void Log(LogType EventType, string ServerProcess, string Message, Exception? ExceptionValue = null)
static public void Log(LogType EventType, string ServerProcess, string Message, Exception? ExceptionValue = null, bool LogToDiskOnly = false)
{
LogItem logItem = new LogItem
{
@@ -20,7 +16,7 @@ namespace gaseous_tools
EventType = EventType,
Process = ServerProcess,
Message = Message,
ExceptionValue = ExceptionValue
ExceptionValue = Common.ReturnValueIfNull(ExceptionValue, "").ToString()
};
bool AllowWrite = false;
@@ -65,67 +61,73 @@ namespace gaseous_tools
Console.WriteLine(TraceOutput);
Console.ResetColor();
Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings
if (LogToDiskOnly == false)
{
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
Formatting = Newtonsoft.Json.Formatting.None
};
serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception) VALUES (@EventTime, @EventType, @Process, @Message, @Exception);";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("EventRententionDate", DateTime.UtcNow.AddDays(Config.LoggingConfiguration.LogRetention * -1));
dbDict.Add("EventTime", logItem.EventTime);
dbDict.Add("EventType", logItem.EventType);
dbDict.Add("Process", logItem.Process);
dbDict.Add("Message", logItem.Message);
dbDict.Add("Exception", Common.ReturnValueIfNull(logItem.ExceptionValue, "").ToString());
// write log file
string JsonOutput = Newtonsoft.Json.JsonConvert.SerializeObject(logItem, serializerSettings);
File.AppendAllText(Config.LogFilePath, JsonOutput);
}
// quick clean before we go
if (LastRetentionClean.AddHours(RetentionCleanInterval) < DateTime.UtcNow)
try
{
LogCleanup();
db.ExecuteCMD(sql, dbDict);
}
catch (Exception ex)
{
LogToDisk(logItem, TraceOutput, ex);
}
}
else
{
LogToDisk(logItem, TraceOutput, null);
}
}
}
static public List<LogItem> GetLogs() {
string logData = File.ReadAllText(Config.LogFilePath);
static void LogToDisk(LogItem logItem, string TraceOutput, Exception? exception)
{
if (exception != null)
{
// dump the error
File.AppendAllText(Config.LogFilePath, logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Process + ": " + logItem.Message + Environment.NewLine + exception.ToString());
// something went wrong writing to the db
File.AppendAllText(Config.LogFilePath, logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": The following event was unable to be written to the log database:");
}
File.AppendAllText(Config.LogFilePath, TraceOutput);
}
static public List<LogItem> GetLogs()
{
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT * FROM ServerLogs ORDER BY Id DESC";
DataTable dataTable = db.ExecuteCMD(sql);
List<LogItem> logs = new List<LogItem>();
if (File.Exists(Config.LogFilePath))
foreach (DataRow row in dataTable.Rows)
{
StreamReader sr = new StreamReader(Config.LogFilePath);
while (!sr.EndOfStream)
LogItem log = new LogItem
{
LogItem logItem = Newtonsoft.Json.JsonConvert.DeserializeObject<LogItem>(sr.ReadLine());
logs.Add(logItem);
}
logs.Reverse();
EventTime = DateTime.Parse(((DateTime)row["EventTime"]).ToString("yyyy-MM-ddThh:mm:ss") + 'Z'),
EventType = (LogType)row["EventType"],
Process = (string)row["Process"],
Message = (string)row["Message"],
ExceptionValue = (string)row["Exception"]
};
logs.Add(log);
}
return logs;
}
static public void LogCleanup()
{
Log(LogType.Information, "Log Cleanup", "Purging log files older than " + Config.LoggingConfiguration.LogRetention + " days");
LastRetentionClean = DateTime.UtcNow;
string[] files = Directory.GetFiles(Config.LogPath, "Server Log *.json");
foreach (string file in files)
{
FileInfo fi = new FileInfo(file);
if (fi.LastAccessTime.AddDays(Config.LoggingConfiguration.LogRetention) < DateTime.Now)
{
try
{
fi.Delete();
}
catch
{
Log(LogType.Warning, "Log Cleanup", "Failed purging log " + fi.FullName);
}
}
}
}
public enum LogType
{
Information = 0,
@@ -151,7 +153,7 @@ namespace gaseous_tools
_Message = value;
}
}
public Exception? ExceptionValue { get; set; }
public string? ExceptionValue { get; set; }
}
}
}