Cache objects from database in memory to improve performance (#151)

* IGDB objects are now cached in memory

* Completed caching of PlatformMaps

gaseous-server/Classes/Metadata/Storage.cs

@@ -4,6 +4,7 @@ using System.Reflection;
 using gaseous_tools;
 using IGDB;
 using IGDB.Models;
+using Microsoft.Extensions.Caching.Memory;
 
 namespace gaseous_server.Classes.Metadata
 {
@@ -16,14 +17,32 @@ namespace gaseous_server.Classes.Metadata
 			Expired
 		}
 
+        private static Dictionary<string, MemoryCacheObject> ObjectCache = new Dictionary<string, MemoryCacheObject>();
+
 		public static CacheStatus GetCacheStatus(string Endpoint, string Slug)
 		{
-			return _GetCacheStatus(Endpoint, "slug", Slug);
+            CacheClean();
+            if (ObjectCache.ContainsKey(Endpoint + Slug))
+            {
+                return CacheStatus.Current;
+            }
+            else
+            {
+			    return _GetCacheStatus(Endpoint, "slug", Slug);
+            }
 		}
 
         public static CacheStatus GetCacheStatus(string Endpoint, long Id)
         {
-			return _GetCacheStatus(Endpoint, "id", Id);
+            CacheClean();
+            if (ObjectCache.ContainsKey(Endpoint + Id))
+            {
+                return CacheStatus.Current;
+            }
+            else
+			{
+                return _GetCacheStatus(Endpoint, "id", Id);
+            }
         }
 
         public static CacheStatus GetCacheStatus(DataRow Row)
@@ -164,6 +183,21 @@ namespace gaseous_server.Classes.Metadata
 		{
             string Endpoint = EndpointType.GetType().Name;
 
+            if (ObjectCache.ContainsKey(Endpoint + SearchValue))
+            {
+                MemoryCacheObject cacheObject = ObjectCache[Endpoint + SearchValue];
+                if (cacheObject.ExpiryTime < DateTime.UtcNow)
+                {
+                    // object has expired, remove it
+                    ObjectCache.Remove(Endpoint + SearchValue);
+                }
+                else
+                {
+                    // object is valid, return it
+                    return (T)cacheObject.Object;
+                }
+            }
+
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 
             string sql = "SELECT * FROM " + Endpoint + " WHERE " + SearchField + " = @" + SearchField;
@@ -181,7 +215,11 @@ namespace gaseous_server.Classes.Metadata
             else
             {
 				DataRow dataRow = dt.Rows[0];
-                return BuildCacheObject<T>(EndpointType, dataRow);
+                object returnObject = BuildCacheObject<T>(EndpointType, dataRow);
+                ObjectCache.Add(Endpoint + SearchValue, new MemoryCacheObject{
+                    Object = returnObject
+                });
+                return (T)returnObject;
             }
         }
 
@@ -380,6 +418,31 @@ namespace gaseous_server.Classes.Metadata
 
             return EndpointType;
         }
+
+        private static void CacheClean()
+        {
+            Dictionary<string, MemoryCacheObject> workCache = ObjectCache;
+            foreach (KeyValuePair<string, MemoryCacheObject> objectCache in workCache)
+            {
+                if (objectCache.Value.ExpiryTime < DateTime.UtcNow)
+                {
+                    ObjectCache.Remove(objectCache.Key);
+                }
+            }
+        }
+
+        private class MemoryCacheObject
+        {
+            public object Object { get; set; }
+            public DateTime CreationTime { get; } = DateTime.UtcNow;
+            public DateTime ExpiryTime 
+            {
+                get
+                {
+                    return CreationTime.AddMinutes(60);
+                }
+            }
+        }
     }
 }
 

gaseous-server/Controllers/LogsController.cs

@@ -13,9 +13,9 @@ namespace gaseous_server.Controllers
     {
         [HttpGet]
         [ProducesResponseType(StatusCodes.Status200OK)]
-        public List<Logging.LogItem> Logs()
+        public List<Logging.LogItem> Logs(long? StartIndex, int PageNumber = 1, int PageSize = 100)
         {
-            return Logging.GetLogs();
+            return Logging.GetLogs(StartIndex, PageNumber, PageSize);
         }
     }
 }

gaseous-server/Models/PlatformMapping.cs

@@ -15,6 +15,8 @@ namespace gaseous_server.Models
 {
 	public class PlatformMapping
 	{
+        private static Dictionary<string, PlatformMapItem> PlatformMapCache = new Dictionary<string, PlatformMapItem>();
+
         /// <summary>
         /// Updates the platform map from the embedded platform map resource
         /// </summary>
@@ -98,7 +100,15 @@ namespace gaseous_server.Models
                 List<PlatformMapItem> platformMaps = new List<PlatformMapItem>();
                 foreach (DataRow row in data.Rows)
                 {
-                    platformMaps.Add(BuildPlatformMapItem(row));
+                    long mapId = (long)row["Id"];
+                    if (PlatformMapCache.ContainsKey(mapId.ToString()))
+                    {
+                        platformMaps.Add(PlatformMapCache[mapId.ToString()]);
+                    }
+                    else
+                    {
+                        platformMaps.Add(BuildPlatformMapItem(row));
+                    }
                 }
 
                 platformMaps.Sort((x, y) => x.IGDBName.CompareTo(y.IGDBName));
@@ -109,23 +119,30 @@ namespace gaseous_server.Models
 
         public static PlatformMapItem GetPlatformMap(long Id)
         {
-            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            if (PlatformMapCache.ContainsKey(Id.ToString()))
-            string sql = "SELECT * FROM PlatformMap WHERE Id = @Id";
-            Dictionary<string, object> dbDict = new Dictionary<string, object>();
-            dbDict.Add("Id", Id);
-            DataTable data = db.ExecuteCMD(sql, dbDict);
-
-            if (data.Rows.Count > 0)
             {
-                PlatformMapItem platformMap = BuildPlatformMapItem(data.Rows[0]);
+                return PlatformMapCache[Id.ToString()];
-
-                return platformMap;
             }
             else
             {
-                Exception exception = new Exception("Platform Map Id " + Id + " does not exist.");
+                Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-                Logging.Log(Logging.LogType.Critical, "Platform Map", "Platform Map Id " + Id + " does not exist.", exception);
+                string sql = "SELECT * FROM PlatformMap WHERE Id = @Id";
-                throw exception;
+                Dictionary<string, object> dbDict = new Dictionary<string, object>();
+                dbDict.Add("Id", Id);
+                DataTable data = db.ExecuteCMD(sql, dbDict);
+
+                if (data.Rows.Count > 0)
+                {
+                    PlatformMapItem platformMap = BuildPlatformMapItem(data.Rows[0]);
+
+                    return platformMap;
+                }
+                else
+                {
+                    Exception exception = new Exception("Platform Map Id " + Id + " does not exist.");
+                    Logging.Log(Logging.LogType.Critical, "Platform Map", "Platform Map Id " + Id + " does not exist.", exception);
+                    throw exception;
+                }
             }
         }
 
@@ -218,6 +235,11 @@ namespace gaseous_server.Models
                     db.ExecuteCMD(sql, dbDict);
                 }
             }
+
+            if (PlatformMapCache.ContainsKey(item.IGDBId.ToString()))
+            {
+                PlatformMapCache.Remove(item.IGDBId.ToString());
+            }
         }
 
         static PlatformMapItem BuildPlatformMapItem(DataRow row)
@@ -321,6 +343,15 @@ namespace gaseous_server.Models
             };
             mapItem.Bios = bioss;
             
+            if (PlatformMapCache.ContainsKey(IGDBId.ToString()))
+            {
+                PlatformMapCache[IGDBId.ToString()] = mapItem;
+            }
+            else
+            {
+                PlatformMapCache.Add(IGDBId.ToString(), mapItem);
+            }
+
             return mapItem;
         }
 

gaseous-server/Program.cs

@@ -182,9 +182,8 @@ ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
     })
     );
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
-    ProcessQueue.QueueItemType.LibraryScan, 30, new List<ProcessQueue.QueueItemType>
+    ProcessQueue.QueueItemType.LibraryScan, 1440, new List<ProcessQueue.QueueItemType>
     {
-        ProcessQueue.QueueItemType.TitleIngestor,
         ProcessQueue.QueueItemType.OrganiseLibrary
     })
     );

gaseous-server/Timer.cs

@@ -36,7 +36,7 @@ namespace gaseous_server
             List<ProcessQueue.QueueItem> ActiveList = new List<ProcessQueue.QueueItem>();
             ActiveList.AddRange(ProcessQueue.QueueItems);
             foreach (ProcessQueue.QueueItem qi in ActiveList) {
-                if (CheckProcessBlockList(qi) == true) {
+                if (CheckIfProcessIsBlockedByOthers(qi) == false) {
                     qi.BlockedState(false);
                     if (DateTime.UtcNow > qi.NextRunTime || qi.Force == true)
                     {
@@ -69,30 +69,24 @@ namespace gaseous_server
             _timer?.Dispose();
         }
 
-        private bool CheckProcessBlockList(ProcessQueue.QueueItem queueItem)
+        private bool CheckIfProcessIsBlockedByOthers(ProcessQueue.QueueItem queueItem)
         {
-            if (queueItem.Blocks.Count > 0)
+            foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
             {
-                foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
+                if (qi.ItemState == ProcessQueue.QueueItemState.Running) {
-                {
+                    // other service is running, check if queueItem is blocked by it
                     if (
-                        (
+                        qi.Blocks.Contains(queueItem.ItemType) ||
-                            queueItem.Blocks.Contains(qi.ItemType) || 
+                        qi.Blocks.Contains(ProcessQueue.QueueItemType.All)
-                            queueItem.Blocks.Contains(ProcessQueue.QueueItemType.All)
+                    )
-                        ) &&
-                            qi.ItemState == ProcessQueue.QueueItemState.Running
-                        )
                     {
-                        return false;
+                        Console.WriteLine(queueItem.ItemType.ToString() + " is blocked by " + qi.ItemType.ToString());
+                        return true;
                     }
                 }
+            }
 
-                return true;
+            return false;
-            }
-            else
-            {
-                return true;
-            }
         }
     }
 }

gaseous-server/wwwroot/pages/settings/about.html

@@ -4,13 +4,17 @@
 
 <table style="width: 100%;">
     <tr>
-        <th>Home Page</th>
+        <th style="width: 20%;">Home Page</th>
         <td><a href="https://github.com/gaseous-project/gaseous-server" class="romlink">https://github.com/gaseous-project/gaseous-server</a></td>
     </tr>
     <tr>
         <th>Bugs and Feature Requests</th>
         <td><a href="https://github.com/gaseous-project/gaseous-server/issues" class="romlink">https://github.com/gaseous-project/gaseous-server/issues</a></td>
     </tr>
+    <tr>
+        <th>Join our Discord</th>
+        <td><a href="https://discord.gg/Nhu7wpT3k4" class="romlink">https://discord.gg/Nhu7wpT3k4</a></td>
+    </tr>
     <tr>
         <th>Server Version</th>
         <td id="settings_appversion"></td>
@@ -19,6 +23,18 @@
         <th>Database Schema Version</th>
         <td id="settings_dbversion"></td>
     </tr>
+    <tr>
+        <td colspan="2">
+            <h3>Projects That Make Gaseous Possible</h3>
+        </td>
+    </tr>
+    <tr>
+        <td style="text-align: center;"><a href="https://github.com/EmulatorJS/EmulatorJS" target="_blank"><img src="/images/EmulatorJS.png" style="height: 36px;" /></a></td>
+        <td>
+            The EmulatorJS Project<br />
+            <a href="https://github.com/EmulatorJS/EmulatorJS" target="_blank" class="romlink">https://github.com/EmulatorJS/EmulatorJS</a>
+        </td>
+    </tr>
     <tr>
         <td colspan="2">
             <h3>Data Sources</h2>
@@ -26,11 +42,12 @@
         </td>
     </tr>
     <tr>
-        <td>
+        <td style="text-align: center;">
             <a href="https://www.igdb.com/" target="_blank"><img src="/images/IGDB_logo.svg" style="filter: invert(100%); height: 36px;" /></a>
         </td>
         <td>
-            The Internet Game Database
+            The Internet Game Database<br />
+            <a href="https://www.igdb.com/" target="_blank" class="romlink">https://www.igdb.com/</a>
         </td>
     </tr>
     <tr>
@@ -39,19 +56,21 @@
         </td>
     </tr>
     <tr>
-        <td>
+        <td style="text-align: center;">
             <a href="https://www.tosecdev.org/" target="_blank"><img src="/images/TOSEC_logo.gif" style="height: 36px;" /></a>
         </td>
         <td>
-            The Old School Emulation Center
+            The Old School Emulation Center<br />
+            <a href="https://www.tosecdev.org/" target="_blank" class="romlink">https://www.tosecdev.org/</a>
         </td>
     </tr>
     <tr>
-        <td>
+        <td style="text-align: center;">
             <a href="https://www.progettosnaps.net/index.php" target="_blank"><img src="/images/ProgettoSnaps.gif" style="height: 36px;" /></a>
         </td>
         <td>
-            Progetto-Snaps
+            Progetto-Snaps<br />
+            <a href="https://www.progettosnaps.net/index.php" target="_blank" class="romlink">https://www.progettosnaps.net/index.php</a>
         </td>
     </tr>
 </table>

gaseous-server/wwwroot/pages/settings/logs.html

@@ -7,30 +7,53 @@
     
 </table>
 
+<div style="width: 960px; text-align: center;">
+    <button value="Load More" onclick="loadLogs(lastStartIndex, currentPage);">Load More</button>
+</div>
+
 <script type="text/javascript">
-    function loadLogs() {
+    var lastStartIndex = 0;
+    var currentPage = 1;
+
+    function loadLogs(StartIndex, PageNumber) {
+        var apiQuery = '';
+
+        if (StartIndex && PageNumber) {
+            currentPage += 1;
+            apiQuery = '?StartIndex=' + StartIndex + '&PageNumber=' + PageNumber;
+        } else {
+            currentPage = 1;
+        }
+
         ajaxCall(
-            '/api/v1/Logs',
+            '/api/v1/Logs' + apiQuery,
             'GET',
             function (result) {
                 var newTable = document.getElementById('settings_events_table');
-                newTable.innerHTML = '';
+                if (currentPage == 1) {
-                newTable.appendChild(
+                    newTable.innerHTML = '';
-                    createTableRow(
+                
-                        true, 
+                    newTable.appendChild(
-                        [ 
+                        createTableRow(
-                            ['Event Time', 'logs_table_cell_150px'],
+                            true, 
-                            ['Severity', 'logs_table_cell_150px'],
+                            [
-                            'Process',
+                                //'Id',
-                            'Message'
+                                ['Event Time', 'logs_table_cell_150px'],
-                        ],
+                                ['Severity', 'logs_table_cell_150px'],
-                        '',
+                                'Process',
-                        ''
+                                'Message'
-                        )
+                            ],
-                    );
+                            '',
+                            ''
+                            )
+                        );
+                }
 
                 for (var i = 0; i < result.length; i++) {
+                    lastStartIndex = result[i].id;
+
                     var newRow = [
+                        //result[i].id,
                         moment(result[i].eventTime).fromNow(),
                         result[i].eventType,
                         result[i].process,

gaseous-server/wwwroot/pages/settings/system.html

@@ -64,7 +64,7 @@
 
                     var itemStateName;
                     var itemLastStart;
-                    if (result[i].IsBlocked == false) {
+                    if (result[i].isBlocked == false) {
                         switch (result[i].itemState) {
                             case 'NeverStarted':
                                 itemStateName = "Never started";

gaseous-tools/Logging.cs

@@ -111,17 +111,30 @@ namespace gaseous_tools
             File.AppendAllText(Config.LogFilePath, TraceOutput);
         }
 
-        static public List<LogItem> GetLogs() 
+        static public List<LogItem> GetLogs(long? StartIndex, int PageNumber = 1, int PageSize = 100) 
         {
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-            string sql = "SELECT * FROM ServerLogs ORDER BY Id DESC";
+            string sql = "";
-            DataTable dataTable = db.ExecuteCMD(sql);
+            if (StartIndex == null)
+            {
+                sql = "SELECT * FROM ServerLogs ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+            }
+            else
+            {
+                sql = "SELECT * FROM ServerLogs WHERE Id < @StartIndex ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+            }
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+            dbDict.Add("StartIndex", StartIndex);
+            dbDict.Add("PageNumber", (PageNumber - 1) * PageSize);
+            dbDict.Add("PageSize", PageSize);
+            DataTable dataTable = db.ExecuteCMD(sql, dbDict);
 
             List<LogItem> logs = new List<LogItem>();
             foreach (DataRow row in dataTable.Rows)
             {
                 LogItem log = new LogItem
                 {
+                    Id = (long)row["Id"],
                     EventTime = DateTime.Parse(((DateTime)row["EventTime"]).ToString("yyyy-MM-ddThh:mm:ss") + 'Z'),
                     EventType = (LogType)row["EventType"],
                     Process = (string)row["Process"],
@@ -145,6 +158,7 @@ namespace gaseous_tools
 
         public class LogItem
         {
+            public long Id { get; set; }
             public DateTime EventTime { get; set; }
             public LogType? EventType { get; set; }
             public string Process { get; set; } = "";