Logs now have filtering options (#219)

* Added logging configuration options * Add support for filtering logs
2023-12-09 14:07:08 +11:00
parent 9e346910f4
commit 84017639eb
8 changed files with 315 additions and 25 deletions
--- a/gaseous-server/Classes/Config.cs
+++ b/gaseous-server/Classes/Config.cs
@@ -503,11 +503,7 @@ namespace gaseous_server.Classes
                // log retention in days
                public int LogRetention = 7;

-                public enum LoggingFormat
-                {
-                    Json,
-                    Text
-                }
+                public bool AlwaysLogToDisk = false;
            }
        }
    }
--- a/gaseous-server/Classes/Logging.cs
+++ b/gaseous-server/Classes/Logging.cs
@@ -7,6 +7,7 @@ namespace gaseous_server.Classes
 {
 	public class Logging
 	{
+        private static DateTime lastDiskRetentionSweep = DateTime.UtcNow;
        public static bool WriteToDiskOnly { get; set; } = false;

        static public void Log(LogType EventType, string ServerProcess, string Message, Exception? ExceptionValue = null, bool LogToDiskOnly = false)
@@ -69,6 +70,11 @@ namespace gaseous_server.Classes

                if (LogToDiskOnly == false)
                {
+                    if (Config.LoggingConfiguration.AlwaysLogToDisk == true)
+                    {
+                        LogToDisk(logItem, TraceOutput, null);
+                    }
+
                    Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
                    string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception) VALUES (@EventTime, @EventType, @Process, @Message, @Exception);";
                    Dictionary<string, object> dbDict = new Dictionary<string, object>();
@@ -93,6 +99,22 @@ namespace gaseous_server.Classes
                    LogToDisk(logItem, TraceOutput, null);
                }
            }
+
+            if (lastDiskRetentionSweep.AddMinutes(60) < DateTime.UtcNow)
+            {
+                // time to delete any old logs
+                lastDiskRetentionSweep = DateTime.UtcNow;
+                string[] files = Directory.GetFiles(Config.LogPath);
+
+                foreach (string file in files)
+                {
+                    FileInfo fi = new FileInfo(file);
+                    if (fi.LastAccessTime < DateTime.Now.AddDays(Config.LoggingConfiguration.LogRetention * -1)) 
+                    { 
+                        fi.Delete(); 
+                    }
+                }
+            }
        }

        static void LogToDisk(LogItem logItem, string TraceOutput, Exception? exception)
@@ -110,22 +132,82 @@ namespace gaseous_server.Classes
            File.AppendAllText(Config.LogFilePath, TraceOutput);
        }

-        static public List<LogItem> GetLogs(long? StartIndex, int PageNumber = 1, int PageSize = 100) 
+        static public List<LogItem> GetLogs(LogsViewModel model) 
        {
            Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+            dbDict.Add("StartIndex", model.StartIndex);
+            dbDict.Add("PageNumber", (model.PageNumber - 1) * model.PageSize);
+            dbDict.Add("PageSize", model.PageSize);
            string sql = "";
-            if (StartIndex == null)
+
+            List<string> whereClauses = new List<string>();
+
+            // handle status criteria
+            if (model.Status != null)
            {
-                sql = "SELECT * FROM ServerLogs ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+                if (model.Status.Count > 0)
+                {
+                    List<string> statusWhere = new List<string>();
+                    for (int i = 0; i < model.Status.Count; i++)
+                    {
+                        string valueName = "@eventtype" + i;
+                        statusWhere.Add(valueName);
+                        dbDict.Add(valueName, (int)model.Status[i]);
+                    }
+
+                    whereClauses.Add("EventType IN (" + string.Join(",", statusWhere) + ")");
+                }
+            }
+
+            // handle start date criteria
+            if (model.StartDateTime != null)
+            {
+                dbDict.Add("startdate", model.StartDateTime);
+                whereClauses.Add("EventTime >= @startdate");
+            }
+
+            // handle end date criteria
+            if (model.EndDateTime != null)
+            {
+                dbDict.Add("enddate", model.EndDateTime);
+                whereClauses.Add("EventTime <= @enddate");
+            }
+
+            // handle search text criteria
+            if (model.SearchText != null)
+            {
+                if (model.SearchText.Length > 0)
+                {
+                    dbDict.Add("messageSearch", model.SearchText);
+                    whereClauses.Add("MATCH(Message) AGAINST (@messageSearch)");
+                }
+            }
+
+            // compile WHERE clause
+            string whereClause = "";
+            if (whereClauses.Count > 0)
+            {
+                whereClause = "(" + String.Join(" AND ", whereClauses) + ")";
+            }
+
+            // execute query
+            if (model.StartIndex == null)
+            {
+                if (whereClause.Length > 0)
+                {
+                    whereClause = "WHERE " + whereClause;
+                }
+                sql = "SELECT * FROM ServerLogs " + whereClause + " ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
            }
            else
            {
-                sql = "SELECT * FROM ServerLogs WHERE Id < @StartIndex ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+                if (whereClause.Length > 0)
+                {
+                    whereClause = "AND " + whereClause;
+                }
+                sql = "SELECT * FROM ServerLogs WHERE Id < @StartIndex " + whereClause + " ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
            }
-            Dictionary<string, object> dbDict = new Dictionary<string, object>();
-            dbDict.Add("StartIndex", StartIndex);
-            dbDict.Add("PageNumber", (PageNumber - 1) * PageSize);
-            dbDict.Add("PageSize", PageSize);
            DataTable dataTable = db.ExecuteCMD(sql, dbDict);

            List<LogItem> logs = new List<LogItem>();
@@ -175,6 +257,17 @@ namespace gaseous_server.Classes
            }
            public string? ExceptionValue { get; set; }
        }
+
+        public class LogsViewModel
+        {
+            public long? StartIndex { get; set; }
+            public int PageNumber { get; set; } = 1;
+            public int PageSize { get; set; } = 100;
+            public List<LogType> Status { get; set; } = new List<LogType>();
+            public DateTime? StartDateTime { get; set; }
+            public DateTime? EndDateTime { get; set; }
+            public string? SearchText { get; set; }
+        }
    }
 }

--- a/gaseous-server/Controllers/V1.0/LogsController.cs
+++ b/gaseous-server/Controllers/V1.0/LogsController.cs
@@ -17,11 +17,11 @@ namespace gaseous_server.Controllers
    {
        [MapToApiVersion("1.0")]
        [MapToApiVersion("1.1")]
-        [HttpGet]
+        [HttpPost]
        [ProducesResponseType(StatusCodes.Status200OK)]
-        public List<Logging.LogItem> Logs(long? StartIndex, int PageNumber = 1, int PageSize = 100)
+        public List<Logging.LogItem> Logs(Logging.LogsViewModel model)
        {
-            return Logging.GetLogs(StartIndex, PageNumber, PageSize);
+            return Logging.GetLogs(model);
        }
    }
 }
--- a/gaseous-server/Controllers/V1.0/SystemController.cs
+++ b/gaseous-server/Controllers/V1.0/SystemController.cs
@@ -179,6 +179,40 @@ namespace gaseous_server.Controllers
            return Ok();
        }

+        [MapToApiVersion("1.0")]
+        [MapToApiVersion("1.1")]
+        [HttpGet]
+        [Route("Settings/System")]
+        [Authorize(Roles = "Admin")]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        public ActionResult GetSystemSettings()
+        {
+            SystemSettingsModel systemSettingsModel = new SystemSettingsModel{
+                AlwaysLogToDisk = Config.LoggingConfiguration.AlwaysLogToDisk,
+                MinimumLogRetentionPeriod = Config.LoggingConfiguration.LogRetention
+            };
+
+            return Ok(systemSettingsModel);
+        }
+
+        [MapToApiVersion("1.0")]
+        [MapToApiVersion("1.1")]
+        [HttpPost]
+        [Route("Settings/System")]
+        [Authorize(Roles = "Admin")]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        public ActionResult SetSystemSettings(SystemSettingsModel model)
+        {
+            if (ModelState.IsValid)
+            {
+                Config.LoggingConfiguration.AlwaysLogToDisk = model.AlwaysLogToDisk;
+                Config.LoggingConfiguration.LogRetention = model.MinimumLogRetentionPeriod;
+                Config.UpdateConfig();
+            }
+
+            return Ok(model);
+        }
+
        private SystemInfo.PathItem GetDisk(string Path)
        {
            SystemInfo.PathItem pathItem = new SystemInfo.PathItem {
@@ -278,4 +312,10 @@ namespace gaseous_server.Controllers
        public int DefaultInterval { get; set; }
        public int MinimumAllowedValue { get; set; }
    }
+
+    public class SystemSettingsModel
+    {
+        public bool AlwaysLogToDisk { get; set; }
+        public int MinimumLogRetentionPeriod { get; set; }
+    }
 }
--- a/gaseous-server/Support/Database/MySQL/gaseous-1007.sql
+++ b/gaseous-server/Support/Database/MySQL/gaseous-1007.sql
@@ -63,4 +63,7 @@ CREATE TABLE `User_Settings` (
  `Id` VARCHAR(128) NOT NULL,
  `Setting` VARCHAR(45) NOT NULL,
  `Value` LONGTEXT NULL DEFAULT NULL,
-  PRIMARY KEY (`Id`, `Setting`));
+  PRIMARY KEY (`Id`, `Setting`));
+
+ALTER TABLE `ServerLogs` 
+ADD FULLTEXT INDEX `ft_message` USING BTREE (`Message`) VISIBLE;
--- a/gaseous-server/wwwroot/pages/settings/logs.html
+++ b/gaseous-server/wwwroot/pages/settings/logs.html
@@ -2,7 +2,34 @@
    <h1 id="gametitle_label">Logs</h1>
 </div>

-<a href="#" class="romlink" onclick="loadLogs();" style="float: right;"><img src="/images/refresh.svg" alt="Refresh" title="Refresh" class="banner_button_image" /></a>
+<table style="width: 960px; max-width: 960px;" cellspacing="0">
+    <tr>
+        <td>
+            <input type="datetime-local" id="logs_startdate" />
+        </td>
+        <td>
+            <input type="datetime-local" id="logs_enddate" />
+        </td>
+        <td>
+            <input type="checkbox" id="logs_type_info"><label for="logs_type_info">Information</label>
+        </td>
+        <td>
+            <input type="checkbox" id="logs_type_warning"><label for="logs_type_warning">Warning</label>
+        </td>
+        <td>
+            <input type="checkbox" id="logs_type_critical"><label for="logs_type_critical">Critical</label>
+        </td>
+        <td>
+            <input type="text" id="logs_textsearch" placeholder="Search" />
+        </td>
+        <td>
+            <button onclick="loadLogs();">Search</button>
+            <button onclick="resetFilters();">Reset</button>
+        </td>
+    </tr>
+</table>
+
+<!-- <a href="#" class="romlink" onclick="loadLogs();" style="float: right;"><img src="/images/refresh.svg" alt="Refresh" title="Refresh" class="banner_button_image" /></a> -->
 <table id="settings_events_table" style="width: 960px; max-width: 960px;" cellspacing="0">
    
 </table>
@@ -14,20 +41,64 @@
 <script type="text/javascript">
    var lastStartIndex = 0;
    var currentPage = 1;
+    var searchModel = {};
+
+    function resetFilters() {
+        document.getElementById('logs_startdate').value = '';
+        document.getElementById('logs_enddate').value = '';
+        document.getElementById('logs_type_info').checked = false;
+        document.getElementById('logs_type_warning').checked = false;
+        document.getElementById('logs_type_critical').checked = false;
+        document.getElementById('logs_textsearch').value = '';
+
+        loadLogs();
+    }

    function loadLogs(StartIndex, PageNumber) {
-        var apiQuery = '';
+        var model = {}

        if (StartIndex && PageNumber) {
            currentPage += 1;
-            apiQuery = '?StartIndex=' + StartIndex + '&PageNumber=' + PageNumber;
+
+            // get saved search model
+            model = searchModel;
+            model.StartIndex = StartIndex;
+            model.PageNumber = PageNumber;
        } else {
            currentPage = 1;
+
+            // create search model
+            var statusList = [];
+            if (document.getElementById('logs_type_info').checked == true) { statusList.push(0); }
+            if (document.getElementById('logs_type_warning').checked == true) { statusList.push(2); }
+            if (document.getElementById('logs_type_critical').checked == true) { statusList.push(3); }
+            var startDate = null;
+            var startDateObj = document.getElementById('logs_startdate');
+            if (startDateObj.value != null) { startDate = new Date(startDateObj.value); }
+            var endDate = null;
+            var endDateObj = document.getElementById('logs_enddate');
+            if (endDateObj.value != null) { endDate = new Date(endDateObj.value); }
+            var searchText = null;
+            var searchTextObj = document.getElementById('logs_textsearch');
+            if (searchTextObj.value != null) { searchText = searchTextObj.value; }
+
+            model = {
+                "StartIndex": StartIndex,
+                "PageNumber": PageNumber,
+                "PageSize": 100,
+                "Status": statusList,
+                "StartDateTime": startDate,
+                "EndDateTime": endDate,
+                "SearchText": searchText
+            }
+            searchModel = model;
        }

+        console.log(model);
+
        ajaxCall(
-            '/api/v1.1/Logs' + apiQuery,
-            'GET',
+            '/api/v1.1/Logs',
+            'POST',
            function (result) {
                var newTable = document.getElementById('settings_events_table');
                if (currentPage == 1) {
@@ -54,7 +125,7 @@

                    var newRow = [
                        //result[i].id,
-                        moment(result[i].eventTime).format("YYYY-MM-DD H:mm:ss"),
+                        moment(result[i].eventTime).format("YYYY-MM-DD h:mm:ss a"),
                        result[i].eventType,
                        result[i].process,
                        result[i].message
@@ -74,7 +145,11 @@
                        newTable.appendChild(exRow);
                    }
                }
-            }
+            },
+            function (error) {
+
+            },
+            JSON.stringify(model)
        );
    }

--- a/gaseous-server/wwwroot/pages/settings/settings.html
+++ b/gaseous-server/wwwroot/pages/settings/settings.html
@@ -17,6 +17,40 @@
 </table>
 <div style="text-align: right;"><button id="settings_tasktimers_default" onclick="defaultTaskTimers();">Reset to Default</button><button id="settings_tasktimers_new" onclick="saveTaskTimers();">Save</button></div>

+<h3>Logging</h3>
+<table cellspacing="0" style="width: 100%;">
+    <tr>
+        <th>
+            Write logs
+        </th>
+        <td>
+            <input type="radio" name="settings_logs_write" id="settings_logs_write_db" value="false" checked="checked"><label for="settings_logs_write_db"> To database only (default)</label>
+        </td>
+    </tr>
+    <tr>
+        <td></td>
+        <td>
+            <input type="radio" name="settings_logs_write" id="settings_logs_write_fs" value="true"><label for="settings_logs_write_fs"> To database and disk</label>
+        </td>
+    </tr>
+    <tr>
+        <td colspan="2">&nbsp;</td>
+    </tr>
+    <tr>
+        <th>
+            Minimum log retention (days):
+        </th>
+        <td>
+            <input type="number" min="1" id="settings_logs_retention" />
+        </td>
+    </tr>
+    <tr>
+        <td colspan="2" style="text-align: right;">
+            <button id="settings_tasktimers_new" onclick="setLoggingSettings();">Save</button>
+        </td>
+    </tr>
+</table>
+
 <script type="text/javascript">
    function drawLibrary() {
        ajaxCall(
@@ -129,6 +163,55 @@
        saveTaskTimers();
    }

+    function getLoggingSettings() {
+        ajaxCall(
+            '/api/v1/System/Settings/System',
+            'GET',
+            function(result) {
+                var optionToSelect = 'settings_logs_write_db';
+                if (result.alwaysLogToDisk == true) {
+                    optionToSelect = 'settings_logs_write_fs';
+                }
+                document.getElementById(optionToSelect).checked = true;
+
+                document.getElementById('settings_logs_retention').value = result.minimumLogRetentionPeriod;
+            }
+        );
+    }
+
+    function setLoggingSettings() {
+        var alwaysLogToDisk = false;
+        if ($("input[type='radio'][name='settings_logs_write']:checked").val() == "true") {
+            alwaysLogToDisk = true;
+        }
+
+        var retention = document.getElementById('settings_logs_retention');
+        var retentionValue = 0;
+        if (retention.value) {
+            retentionValue = retention.value;
+        } else {
+            retentionValue = 7;
+        }
+
+        var model = {
+            "alwaysLogToDisk": alwaysLogToDisk,
+            "minimumLogRetentionPeriod": retentionValue
+        };
+
+        ajaxCall(
+            '/api/v1/System/Settings/System',
+            'POST',
+            function(result) {
+                getLoggingSettings();
+            },
+            function(error) {
+                getLoggingSettings();
+            },
+            JSON.stringify(model)
+        );
+    }
+
    drawLibrary();
    getBackgroundTaskTimers();
+    getLoggingSettings();
 </script>
--- a/gaseous-server/wwwroot/styles/style.css
+++ b/gaseous-server/wwwroot/styles/style.css
@@ -223,7 +223,7 @@ h3 {
    z-index: 1;
 }

-input[type='text'], input[type='number'], input[type="email"], input[type="password"] {
+input[type='text'], input[type='number'], input[type="email"], input[type="password"], input[type="datetime-local"] {
    background-color: #2b2b2b;
    color: white;
    padding: 5px;