Added log correlation and revised background tasks layout (#230)

gaseous-server/Classes/Common.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Security.Cryptography;
 
 namespace gaseous_server.Classes
@@ -111,5 +112,30 @@ namespace gaseous_server.Classes
 					.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
 		}
     }
+
+	/// <summary>
+    /// Provides a way to set contextual data that flows with the call and 
+    /// async context of a test or invocation.
+    /// </summary>
+    public static class CallContext
+    {
+        static ConcurrentDictionary<string, AsyncLocal<object>> state = new ConcurrentDictionary<string, AsyncLocal<object>>();
+
+        /// <summary>
+        /// Stores a given object and associates it with the specified name.
+        /// </summary>
+        /// <param name="name">The name with which to associate the new item in the call context.</param>
+        /// <param name="data">The object to store in the call context.</param>
+        public static void SetData(string name, object data) =>
+            state.GetOrAdd(name, _ => new AsyncLocal<object>()).Value = data;
+
+        /// <summary>
+        /// Retrieves an object with the specified name from the <see cref="CallContext"/>.
+        /// </summary>
+        /// <param name="name">The name of the item in the call context.</param>
+        /// <returns>The object in the call context associated with the specified name, or <see langword="null"/> if not found.</returns>
+        public static object GetData(string name) =>
+            state.TryGetValue(name, out AsyncLocal<object> data) ? data.Value : null;
+    }
 }
 

gaseous-server/Classes/Logging.cs

@@ -75,8 +75,28 @@ namespace gaseous_server.Classes
                         LogToDisk(logItem, TraceOutput, null);
                     }
 
+                    string correlationId;
+                    if (CallContext.GetData("CorrelationId").ToString() == null)
+                    {
+                        correlationId = "";
+                    }
+                    else
+                    {
+                        correlationId = CallContext.GetData("CorrelationId").ToString();
+                    }
+
+                    string callingProcess;
+                    if (CallContext.GetData("CallingProcess").ToString() == null)
+                    {
+                        callingProcess = "";
+                    }
+                    else
+                    {
+                        callingProcess = CallContext.GetData("CallingProcess").ToString();
+                    }
+
                     Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-                    string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception) VALUES (@EventTime, @EventType, @Process, @Message, @Exception);";
+                    string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception, CorrelationId, CallingProcess) VALUES (@EventTime, @EventType, @Process, @Message, @Exception, @correlationid, @callingprocess);";
                     Dictionary<string, object> dbDict = new Dictionary<string, object>();
                     dbDict.Add("EventRententionDate", DateTime.UtcNow.AddDays(Config.LoggingConfiguration.LogRetention * -1));
                     dbDict.Add("EventTime", logItem.EventTime);
@@ -84,6 +104,8 @@ namespace gaseous_server.Classes
                     dbDict.Add("Process", logItem.Process);
                     dbDict.Add("Message", logItem.Message);
                     dbDict.Add("Exception", Common.ReturnValueIfNull(logItem.ExceptionValue, "").ToString());
+                    dbDict.Add("correlationid", correlationId);
+                    dbDict.Add("callingprocess", callingProcess);
 
                     try
                     {
@@ -184,6 +206,24 @@ namespace gaseous_server.Classes
                 }
             }
 
+            if (model.CorrelationId != null)
+            {
+                if (model.CorrelationId.Length > 0)
+                {
+                    dbDict.Add("correlationId", model.CorrelationId);
+                    whereClauses.Add("CorrelationId = @correlationId");
+                }
+            }
+
+            if (model.CallingProcess != null)
+            {
+                if (model.CallingProcess.Length > 0)
+                {
+                    dbDict.Add("callingProcess", model.CallingProcess);
+                    whereClauses.Add("CallingProcess = @callingProcess");
+                }
+            }
+
             // compile WHERE clause
             string whereClause = "";
             if (whereClauses.Count > 0)
@@ -220,7 +260,9 @@ namespace gaseous_server.Classes
                     EventType = (LogType)row["EventType"],
                     Process = (string)row["Process"],
                     Message = (string)row["Message"],
-                    ExceptionValue = (string)row["Exception"]
+                    ExceptionValue = (string)row["Exception"],
+                    CorrelationId = (string)row["CorrelationId"],
+                    CallingProcess = (string)row["CallingProcess"]
                 };
 
                 logs.Add(log);
@@ -243,6 +285,8 @@ namespace gaseous_server.Classes
             public DateTime EventTime { get; set; }
             public LogType? EventType { get; set; }
             public string Process { get; set; } = "";
+            public string CorrelationId { get; set; } = "";
+            public string? CallingProcess { get; set; } = "";
             private string _Message = "";
             public string Message
             {
@@ -267,6 +311,8 @@ namespace gaseous_server.Classes
             public DateTime? StartDateTime { get; set; }
             public DateTime? EndDateTime { get; set; }
             public string? SearchText { get; set; }
+            public string? CorrelationId { get; set; }
+            public string? CallingProcess { get; set; }
         }
     }
 }

gaseous-server/Classes/SignatureIngestors/XML.cs

@@ -33,6 +33,12 @@ namespace gaseous_server.SignatureIngestors.XML
 
                 SetStatus(i + 1, PathContents.Length, "Processing signature file: " + XMLFile);
 
+                if (Common.SkippableFiles.Contains(Path.GetFileName(XMLFile), StringComparer.OrdinalIgnoreCase))
+                {
+                    Logging.Log(Logging.LogType.Information, "Signature Ingestor - XML", "Skipping file: " + XMLFile);
+                }
+                else
+                {
                     // check xml file md5
                     Common.hashObject hashObject = new Common.hashObject(XMLFile);
                     sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
@@ -249,6 +255,7 @@ namespace gaseous_server.SignatureIngestors.XML
                         Logging.Log(Logging.LogType.Debug, "Signature Ingestor - XML", "Rejecting already imported file: " + XMLFile);
                     }
                 }
+            }
             ClearStatus();
         }
     }

gaseous-server/ProcessQueue.cs

@@ -1,4 +1,6 @@
 using System;
+using System.ComponentModel.Design.Serialization;
+using System.Data;
 using gaseous_server.Classes;
 
 namespace gaseous_server
@@ -56,6 +58,7 @@ namespace gaseous_server
             private bool _AllowManualStart = true;
             private bool _RemoveWhenStopped = false;
             private bool _IsBlocked = false;
+            private string _CorrelationId = "";
             private List<QueueItemType> _Blocks = new List<QueueItemType>();
 
             public QueueItemType ItemType => _ItemType;
@@ -90,6 +93,7 @@ namespace gaseous_server
             public object? Options { get; set; } = null;
             public string CurrentState { get; set; } = "";
             public string CurrentStateProgress { get; set; } = "";
+            public string CorrelationId => _CorrelationId;
             public List<QueueItemType> Blocks => _Blocks;
 
             public void Execute()
@@ -104,7 +108,14 @@ namespace gaseous_server
                         _LastResult = "";
                         _LastError = null;
 
-                        Logging.Log(Logging.LogType.Debug, "Timered Event", "Executing " + _ItemType);
+                        // set the correlation id
+                        Guid correlationId = Guid.NewGuid();
+                        _CorrelationId = correlationId.ToString();
+                        CallContext.SetData("CorrelationId", correlationId);
+                        CallContext.SetData("CallingProcess", _ItemType.ToString());
+
+                        // log the start
+                        Logging.Log(Logging.LogType.Debug, "Timered Event", "Executing " + _ItemType + " with correlation id " + _CorrelationId);
 
                         try
                         {
@@ -238,6 +249,66 @@ namespace gaseous_server
             {
                 _IsBlocked = BlockState;
             }
+
+            public HasErrorsItem HasErrors
+            {
+                get
+                {
+                    return new HasErrorsItem(_CorrelationId);
+                }
+            }
+
+            public class HasErrorsItem
+            {
+                public HasErrorsItem(string? CorrelationId)
+                {
+                    if (CorrelationId != null)
+                    {
+                        if (CorrelationId.Length > 0)
+                        {
+                            Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+                            string sql = "SELECT EventType, COUNT(EventType) AS EventTypes FROM gaseous.ServerLogs WHERE CorrelationId = @correlationid GROUP BY EventType ORDER BY EventType DESC LIMIT 1;";
+                            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+                            dbDict.Add("correlationid", CorrelationId);
+
+                            DataTable data = db.ExecuteCMD(sql, dbDict);
+
+                            if (data.Rows.Count == 0)
+                            {
+                                ErrorType = null;
+                                ErrorCount = 0;
+                            }
+                            else
+                            {
+                                Logging.LogType errorType = (Logging.LogType)data.Rows[0]["EventType"];
+                                if (errorType != Logging.LogType.Information)
+                                {
+                                    ErrorType = errorType;
+                                    ErrorCount = (int)(long)data.Rows[0]["EventTypes"];
+                                }
+                                else
+                                {
+                                    ErrorType = null;
+                                    ErrorCount = 0;
+                                }
+                            }
+                        }
+                        else
+                        {
+                            ErrorType = null;
+                            ErrorCount = 0;
+                        }
+                    }
+                    else
+                    {
+                        ErrorType = null;
+                        ErrorCount = 0;
+                    }
+                }
+
+                public Logging.LogType? ErrorType { get; set; }
+                public int ErrorCount { get; set; }
+            }
         }
 
         public enum QueueItemType

gaseous-server/Support/Database/MySQL/gaseous-1008.sql

@@ -18,3 +18,9 @@ ADD INDEX `idx_SecondaryColumn` (`PlayerPerspectivesId` ASC) VISIBLE;
 
 ALTER TABLE `Relation_Game_Themes` 
 ADD INDEX `idx_SecondaryColumn` (`ThemesId` ASC) VISIBLE;
+
+ALTER TABLE `ServerLogs` 
+ADD COLUMN `CorrelationId` VARCHAR(45) NULL AFTER `Exception`,
+ADD COLUMN `CallingProcess` VARCHAR(45) NULL AFTER `CorrelationId`,
+ADD INDEX `idx_CorrelationId` (`CorrelationId` ASC) VISIBLE,
+ADD INDEX `idx_CallingProcess` (`CallingProcess` ASC) VISIBLE;

gaseous-server/wwwroot/index.html

@@ -3,7 +3,7 @@
 <head>
     <meta charset="utf-8" />
     <script src="/api/v1.1/System/VersionFile"></script>
-    <link type="text/css" rel="stylesheet" dat-href="/styles/style.css" />
+    <link type="text/css" rel="stylesheet" href="/styles/style.css" dat-href="/styles/style.css" />
     <script src="/scripts/jquery-3.6.0.min.js"></script>
     <script src="/scripts/moment-with-locales.min.js"></script>
     <link href="/styles/select2.min.css" rel="stylesheet" />

gaseous-server/wwwroot/pages/settings.html

@@ -47,6 +47,10 @@
     SelectTab(selectedTab);
 
     function SelectTab(TabName) {
+        if (selectedTab != TabName) {
+            window.location.href = '/index.html?page=settings&sub=' + TabName;
+        }
+
         var tocs = document.getElementsByName('properties_toc_item');
         for (var i = 0; i < tocs.length; i++) {
             if ((tocs[i].id) == ("properties_toc_" + TabName)) {

gaseous-server/wwwroot/pages/settings/logs.html

@@ -5,10 +5,7 @@
 <table style="width: 960px; max-width: 960px;" cellspacing="0">
     <tr>
         <td>
-            <input type="datetime-local" id="logs_startdate" />
+            <input type="datetime-local" id="logs_startdate" style="width: 30%;" /> <input type="datetime-local" id="logs_enddate" style="width: 30%;" />
-        </td>
-        <td>
-            <input type="datetime-local" id="logs_enddate" />
         </td>
         <td>
             <input type="checkbox" id="logs_type_info"><label for="logs_type_info">Information</label>
@@ -19,10 +16,17 @@
         <td>
             <input type="checkbox" id="logs_type_critical"><label for="logs_type_critical">Critical</label>
         </td>
-        <td>
+    </tr>
-            <input type="text" id="logs_textsearch" placeholder="Search" />
+    <tr>
+        <td colspan="1">
+            <input type="text" id="logs_textsearch" placeholder="Search" style="width: 75%;" />
         </td>
-        <td>
+        <td colspan="3">
+            <input type="text" id="logs_correlationid" placeholder="Correlation Id" style="width: 75%;" />
+        </td>
+    </tr>
+    <tr>
+        <td colspan="4" style="text-align: right;">
             <button onclick="loadLogs();">Search</button>
             <button onclick="resetFilters();">Reset</button>
         </td>
@@ -43,6 +47,13 @@
     var currentPage = 1;
     var searchModel = {};
 
+    var correlationIdParam = getQueryString('correlationid', 'string');
+    if (correlationIdParam) {
+        if (correlationIdParam.length > 0) {
+            document.getElementById('logs_correlationid').value = correlationIdParam;
+        }
+    }
+
     function resetFilters() {
         document.getElementById('logs_startdate').value = '';
         document.getElementById('logs_enddate').value = '';
@@ -50,6 +61,7 @@
         document.getElementById('logs_type_warning').checked = false;
         document.getElementById('logs_type_critical').checked = false;
         document.getElementById('logs_textsearch').value = '';
+        document.getElementById('logs_correlationid').value = '';
 
         loadLogs();
     }
@@ -81,6 +93,9 @@
             var searchText = null;
             var searchTextObj = document.getElementById('logs_textsearch');
             if (searchTextObj.value != null) { searchText = searchTextObj.value; }
+            var correlationId = null;
+            var correlationIdTextObj = document.getElementById('logs_correlationid');
+            if (correlationIdTextObj.value != null) { correlationId = correlationIdTextObj.value; }
 
             model = {
                 "StartIndex": StartIndex,
@@ -89,7 +104,8 @@
                 "Status": statusList,
                 "StartDateTime": startDate,
                 "EndDateTime": endDate,
-                "SearchText": searchText
+                "SearchText": searchText,
+                "CorrelationId": correlationId
             }
             searchModel = model;
         }
@@ -131,7 +147,7 @@
                         result[i].message
                     ];
 
-                    newTable.appendChild(createTableRow(false, newRow, 'romrow logs_table_row_' + result[i].eventType, 'romcell logs_table_cell'));
+                    newTable.appendChild(createTableRow(false, newRow, 'logs_table_row_' + result[i].eventType, 'romcell logs_table_cell'));
 
                     if (result[i].exceptionValue) {
                         var exceptionString = "<h3>Exception</h3><pre class='logs_table_exception' style='width: 795px; word-wrap: break-word; overflow-wrap: break-word; overflow-y: scroll;'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br />    ") + "</pre>";

gaseous-server/wwwroot/pages/settings/system.html

@@ -31,14 +31,23 @@
             var newTable = document.createElement('table');
             newTable.className = 'romtable';
             newTable.setAttribute('cellspacing', 0);
-            newTable.appendChild(createTableRow(true, ['Task', 'Status', 'Interval', 'Last Run Start', 'Last Run Duration (seconds)', 'Next Run Start', '']));
+            newTable.appendChild(createTableRow(true, ['Task', 'Status', 'Interval<br/>(minutes)', 'Last Run Duration<br />(hh:mm:ss)', '', 'Last Run Start', 'Next Run Start', '']));
 
             if (result) {
                 for (var i = 0; i < result.length; i++) {
+                    if (result[i].itemState != "Disabled") {
                         var itemTypeName = GetTaskFriendlyName(result[i].itemType, result[i].options);
                         
                         var itemStateName;
                         var itemLastStart;
+                        
+                        var hasError = "";
+                        if (result[i].hasErrors) {
+                            if (result[i].hasErrors.errorType != null) {
+                                hasError = " (" + result[i].hasErrors.errorType + ")";
+                            }
+                        }
+
                         if (result[i].isBlocked == false) {
                             switch (result[i].itemState) {
                                 case 'NeverStarted':
@@ -64,9 +73,11 @@
                             }
                         } else {
                             itemStateName = "Blocked";
-                        itemLastStart = moment(result[i].lastRunTime).fromNow();
+                            itemLastStart = moment(result[i].lastRunTime).format("YYYY-MM-DD h:mm:ss a");
                         }
 
+                        itemStateName += hasError;
+
                         var itemInterval = result[i].interval;
                         var nextRunTime = moment(result[i].nextRunTime).format("YYYY-MM-DD h:mm:ss a");
                         var startButton = '';
@@ -81,18 +92,25 @@
                             nextRunTime = '';
                         }
 
+                        var logLink = '';
+                        if (result[i].correlationId) {
+                            logLink = '<a href="/index.html?page=settings&sub=logs&correlationid=' + result[i].correlationId + '" class="romlink">View Log</a>';
+                        }
+
                         var newRow = [
                             itemTypeName,
                             itemStateName,
                             itemInterval,
+                            new Date(result[i].lastRunDuration * 1000).toISOString().slice(11, 19),
+                            logLink,
                             itemLastStart,
-                        result[i].lastRunDuration,
                             nextRunTime,
                             startButton
                         ];
                         newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell'));
                     }
                 }
+            }
 
             var targetDiv = document.getElementById('system_tasks');
             targetDiv.innerHTML = '';

gaseous-server/wwwroot/styles/style.css

@@ -1060,22 +1060,38 @@ button:disabled {
     vertical-align: top;
 }
 
-.logs_table_row_Information:hover {
+.logs_table_row_Information:nth-child(even) {
     background: rgba(42, 41, 150, 0.3);
 }
 
-.logs_table_row_Warning:hover {
+.logs_table_row_Information:nth-child(odd) {
+    background: rgba(10, 9, 83, 0.3);
+}
+
+.logs_table_row_Warning:nth-child(even) {
     background: rgba(139, 150, 41, 0.3);
 }
 
-.logs_table_row_Critical:hover {
+.logs_table_row_Warning:nth-child(odd) {
+    background: rgba(49, 53, 14, 0.3);
+}
+
+.logs_table_row_Critical:nth-child(even) {
     background: rgba(150, 41, 41, 0.3);
 }
 
-.logs_table_row_Debug:hover {
+.logs_table_row_Critical:nth-child(odd) {
+    background: rgba(58, 16, 16, 0.3);
+}
+
+.logs_table_row_Debug:nth-child(even) {
     background: rgba(150, 41, 135, 0.3);
 }
 
+.logs_table_row_Debug:nth-child(odd) {
+    background: rgba(68, 18, 61, 0.3);
+}
+
 .logs_table_exception {
     margin-right: 10px;
     padding: 5px;