Surface logs in the UI (#91)

* JSON type log files now have the extension "json"

* Added logging to collection building

* Logs can now be viewed in the UI, improved log handling

gaseous-server/Classes/Collections.cs

@@ -111,6 +111,7 @@ namespace gaseous_server.Classes
             string CollectionZipFile = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, Id + ".zip");
             if (File.Exists(CollectionZipFile))
             {
+                Logging.Log(Logging.LogType.Warning, "Collections", "Deleting existing build of collection: " + item.Name);
                 File.Delete(CollectionZipFile);
             }
 
@@ -265,6 +266,7 @@ namespace gaseous_server.Classes
 
             CollectionContents collectionContents = new CollectionContents();
             collectionContents.Collection = collectionPlatformItems;
+
             return collectionContents;
         }
 
@@ -277,6 +279,8 @@ namespace gaseous_server.Classes
             {
                 if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild)
                 {
+                    Logging.Log(Logging.LogType.Information, "Collections", "Beginning build of collection: " + collectionItem.Name);
+
                     // set starting
                     string sql = "UPDATE RomCollections SET BuiltStatus=@bs WHERE Id=@id";
                     Dictionary<string, object> dbDict = new Dictionary<string, object>();
@@ -294,6 +298,7 @@ namespace gaseous_server.Classes
                         // clean up if needed
                         if (File.Exists(ZipFilePath))
                         {
+                            Logging.Log(Logging.LogType.Warning, "Collections", "Deleting existing build of collection: " + collectionItem.Name);
                             File.Delete(ZipFilePath);
                         }
 
@@ -321,6 +326,7 @@ namespace gaseous_server.Classes
                                 {
                                     if (File.Exists(biosItem.biosPath))
                                     {
+                                        Logging.Log(Logging.LogType.Information, "Collections", "Copying BIOS file: " + biosItem.filename);
                                         File.Copy(biosItem.biosPath, Path.Combine(ZipBiosPath, biosItem.filename));
                                     }
                                 }
@@ -377,6 +383,7 @@ namespace gaseous_server.Classes
                                 {
                                     if (File.Exists(gameRomItem.Path))
                                     {
+                                        Logging.Log(Logging.LogType.Information, "Collections", "Copying ROM: " + gameRomItem.Name);
                                         File.Copy(gameRomItem.Path, Path.Combine(ZipGamePath, gameRomItem.Name));
                                     }
                                 }
@@ -384,11 +391,13 @@ namespace gaseous_server.Classes
                         }
 
                         // compress to zip
+                        Logging.Log(Logging.LogType.Information, "Collections", "Compressing collection");
                         ZipFile.CreateFromDirectory(ZipFileTempPath, ZipFilePath, CompressionLevel.SmallestSize, false);
 
                         // clean up
                         if (Directory.Exists(ZipFileTempPath))
                         {
+                            Logging.Log(Logging.LogType.Information, "Collections", "Cleaning up");
                             Directory.Delete(ZipFileTempPath, true);
                         }
 

gaseous-server/Controllers/LogsController.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using gaseous_tools;
+using Microsoft.AspNetCore.Mvc;
+
+namespace gaseous_server.Controllers
+{
+    [ApiController]
+    [Route("api/v1/[controller]")]
+    public class LogsController : Controller
+    {
+        [HttpGet]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        public List<Logging.LogItem> Logs()
+        {
+            return Logging.GetLogs();
+        }
+    }
+}

gaseous-server/Program.cs

@@ -16,6 +16,8 @@ db.InitDB();
 
 // load app settings
 Config.InitSettings();
+// write updated settings back to the config file
+Config.UpdateConfig();
 
 // set initial values
 Guid APIKey = Guid.NewGuid();
@@ -158,7 +160,7 @@ ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
     );
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.MetadataRefresh, 360));
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
-    ProcessQueue.QueueItemType.OrganiseLibrary, 2040, new List<ProcessQueue.QueueItemType>
+    ProcessQueue.QueueItemType.OrganiseLibrary, 1440, new List<ProcessQueue.QueueItemType>
     {
         ProcessQueue.QueueItemType.LibraryScan,
         ProcessQueue.QueueItemType.TitleIngestor

gaseous-server/wwwroot/pages/settings.html

@@ -7,6 +7,7 @@
         <div class="filter_header">Settings</div>
         <div id="properties_toc_system" name="properties_toc_item" onclick="SelectTab('system');">System</div>
         <div id="properties_toc_bios" name="properties_toc_item" onclick="SelectTab('bios');">Firmware</div>
+        <div id="properties_toc_logs" name="properties_toc_item" onclick="SelectTab('logs');">Logs</div>
         <div id="properties_toc_about" name="properties_toc_item" onclick="SelectTab('about');">About</div>
     </div>
     <div id="properties_bodypanel">

gaseous-server/wwwroot/pages/settings/logs.html

@@ -0,0 +1,50 @@
+<div id="gametitle">
+    <h1 id="gametitle_label">Logs</h1>
+</div>
+
+<a href="#" class="romlink" onclick="loadLogs();" style="float: right;"><img src="/images/refresh.svg" alt="Refresh" title="Refresh" class="banner_button_image" /></a>
+<table id="settings_events_table" style="width: 100%;" cellspacing="0">
+    
+</table>
+
+<script type="text/javascript">
+    function loadLogs() {
+        ajaxCall(
+            '/api/v1/Logs',
+            'GET',
+            function (result) {
+                var newTable = document.getElementById('settings_events_table');
+                newTable.innerHTML = '';
+                newTable.appendChild(
+                    createTableRow(
+                        true, 
+                        [ 
+                            ['Event Time', 'logs_table_cell_150px'],
+                            ['Severity', 'logs_table_cell_150px'],
+                            'Message'
+                        ],
+                        '',
+                        ''
+                        )
+                    );
+
+                for (var i = 0; i < result.length; i++) {
+                    var exceptionString = '';
+                    if (result[i].exceptionValue) {
+                        exceptionString = "<h3>Exception</h3><pre class='logs_table_exception'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)) + "</pre>";
+                    }
+
+                    var newRow = [
+                        moment(result[i].eventTime).fromNow(),
+                        result[i].eventType,
+                        result[i].process + " - " + result[i].message + exceptionString
+                    ];
+
+                    newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell logs_table_cell'));
+                }
+            }
+        );
+    }
+
+    loadLogs();
+</script>

gaseous-server/wwwroot/scripts/main.js

@@ -259,4 +259,27 @@ function DropDownRenderGameOption(state) {
         );
     }
     return response;
+}
+
+function syntaxHighlight(json) {
+    json = json.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+    return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?|(\{|\})?|(\[|\])?\b)/g, function (match) {
+        var cls = 'number';
+        if (/^"/.test(match)) {
+            if (/:$/.test(match)) {
+                cls = 'key';
+            } else {
+                cls = 'string';
+            }
+        } else if (/true|false/.test(match)) {
+            cls = 'boolean';
+        } else if (/null/.test(match)) {
+            cls = 'null';
+        } else if (/\{|\}/.test(match)) {
+            cls = 'brace';
+        } else if (/\[|\]/.test(match)) {
+            cls = 'square';
+        }
+        return '<span class="' + cls + '">' + match + '</span>';
+    });
 }

gaseous-server/wwwroot/styles/style.css

@@ -877,4 +877,29 @@ button:disabled {
 
 .bgalt1 {
     background-color: transparent;;
-}
+}
+
+.logs_table_cell_150px {
+    width: 150px;
+}
+
+.logs_table_cell {
+    vertical-align: top;
+}
+
+.logs_table_exception {
+    margin-right: 10px;
+    padding: 5px;
+    border-radius: 10px;
+    border-color: #383838;
+    border-width: 1px;
+    background-color: #383838;
+}
+
+.string { color: lightblue; }
+.number { color: lightblue; }
+.boolean { color: lightblue; }
+.null { color: magenta; }
+.key { color: greenyellow; }
+.brace { color: #888; }
+.square { color: #fff000; }

gaseous-tools/Config.cs

@@ -74,7 +74,9 @@ namespace gaseous_tools
         {
             get
             {
-                string logPathName = Path.Combine(LogPath, "Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + ".txt");
+                string logFileExtension = "json";
+                
+                string logPathName = Path.Combine(LogPath, "Server Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + "." + logFileExtension);
                 return logPathName;
             }
         }
@@ -458,7 +460,7 @@ namespace gaseous_tools
             {
                 public bool DebugLogging = false;
 
-                public LoggingFormat LogFormat = Logging.LoggingFormat.Json;
+                public int LogRetention = 30;
 
                 public enum LoggingFormat
                 {

gaseous-tools/Logging.cs

@@ -1,15 +1,24 @@
 using System;
+using System.Diagnostics;
+using System.Reflection;
+using System.Reflection.Metadata.Ecma335;
+using Org.BouncyCastle.Utilities;
 namespace gaseous_tools
 {
 	public class Logging
 	{
-        static public void Log(LogType EventType, string Section, string Message, Exception? ExceptionValue = null)
+        // when was the last clean
+        static DateTime LastRetentionClean = DateTime.UtcNow;
+        // how often to clean in hours
+        const int RetentionCleanInterval = 1;
+
+        static public void Log(LogType EventType, string ServerProcess, string Message, Exception? ExceptionValue = null)
         {
             LogItem logItem = new LogItem
             {
                 EventTime = DateTime.UtcNow,
                 EventType = EventType,
-                Section = Section,
+                Process = ServerProcess,
                 Message = Message,
                 ExceptionValue = ExceptionValue
             };
@@ -30,32 +39,85 @@ namespace gaseous_tools
             if (AllowWrite == true)
             {
                 // console output
-                string TraceOutput = logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Section + ": " + logItem.Message;
+                string TraceOutput = logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Process + ": " + logItem.Message;
                 if (logItem.ExceptionValue != null)
                 {
                     TraceOutput += Environment.NewLine + logItem.ExceptionValue.ToString();
                 }
-                Console.WriteLine(TraceOutput);
+                switch(logItem.EventType) {
-
+                    case LogType.Information:
-                StreamWriter LogFile = File.AppendText(Config.LogFilePath);
+                        Console.ForegroundColor = ConsoleColor.Blue;
-                switch (Config.LoggingConfiguration.LogFormat)
-                {
-                    case Config.ConfigFile.Logging.LoggingFormat.Text:
-                        LogFile.WriteLine(TraceOutput);
                         break;
 
-                    case Config.ConfigFile.Logging.LoggingFormat.Json:
+                    case LogType.Warning:
-                        Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings
+                        Console.ForegroundColor = ConsoleColor.Yellow;
-                        {
-                            NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
-                            Formatting = Newtonsoft.Json.Formatting.Indented
-                        };
-                        serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
-                        string JsonOutput = Newtonsoft.Json.JsonConvert.SerializeObject(logItem, serializerSettings);
-                        LogFile.WriteLine(JsonOutput);
                         break;
+
+                    case LogType.Critical:
+                        Console.ForegroundColor = ConsoleColor.Red;
+                        break;
+
+                    case LogType.Debug:
+                        Console.ForegroundColor = ConsoleColor.Magenta;
+                        break;
+
+                }
+                Console.WriteLine(TraceOutput);
+                Console.ResetColor();
+
+                Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings
+                {
+                    NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
+                    Formatting = Newtonsoft.Json.Formatting.None
+                };
+                serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
+
+                // write log file
+                string JsonOutput = Newtonsoft.Json.JsonConvert.SerializeObject(logItem, serializerSettings);
+                StreamWriter jsonLogFile = File.AppendText(Config.LogFilePath);
+                jsonLogFile.WriteLine(JsonOutput);
+                jsonLogFile.Close();
+            }
+
+            // quick clean before we go
+            if (LastRetentionClean.AddHours(RetentionCleanInterval) < DateTime.UtcNow)
+            {
+                LogCleanup();
+            }
+        }
+
+        static public List<LogItem> GetLogs() {
+            string logData = File.ReadAllText(Config.LogFilePath);
+
+            List<LogItem> logs = new List<LogItem>();
+            if (File.Exists(Config.LogFilePath))
+            {
+                StreamReader sr = new StreamReader(Config.LogFilePath);
+                while (!sr.EndOfStream)
+                {
+                    LogItem logItem = Newtonsoft.Json.JsonConvert.DeserializeObject<LogItem>(sr.ReadLine());
+                    logs.Add(logItem);
+                }
+                logs.Reverse();
+            }
+
+            return logs;
+        }
+
+        static public void LogCleanup()
+        {
+            Log(LogType.Information, "Log Cleanup", "Purging log files older than " + Config.LoggingConfiguration.LogRetention + " days");
+            LastRetentionClean = DateTime.UtcNow;
+
+            string[] files = Directory.GetFiles(Config.LogPath, "Server Log *.json");
+
+            foreach (string file in files)
+            {
+                FileInfo fi = new FileInfo(file);
+                if (fi.LastAccessTime.AddDays(Config.LoggingConfiguration.LogRetention) < DateTime.Now)
+                {
+                    fi.Delete();
                 }
-                LogFile.Close();
             }
         }
 
@@ -70,19 +132,8 @@ namespace gaseous_tools
         public class LogItem
         {
             public DateTime EventTime { get; set; }
-            public LogType EventType { get; set; }
+            public LogType? EventType { get; set; }
-            private string _Section = "";
+            public string Process { get; set; } = "";
-            public string Section
-            {
-                get
-                {
-                    return _Section;
-                }
-                set
-                {
-                    _Section = value;
-                }
-            }
             private string _Message = "";
             public string Message
             {