Overhaul of SQL queries to (hopefully) improve performance with large libraries (#233)

* Latest round of performance updates

* Improved first set up logging

* Updated logging display

gaseous-server/Classes/Collections.cs

@@ -220,7 +220,7 @@ namespace gaseous_server.Classes
                 }
             } else {
                 // get all platforms to pull from
-                Dictionary<string, object> FilterDict = Filters.Filter(AgeRatings.AgeGroups.AgeRestrictionGroupings.Adult, true);
+                Dictionary<string, List<Filters.FilterItem>> FilterDict = Filters.Filter(AgeRatings.AgeGroups.AgeRestrictionGroupings.Adult, true);
                 List<Classes.Filters.FilterItem> filteredPlatforms = (List<Classes.Filters.FilterItem>)FilterDict["platforms"];
                 foreach (Filters.FilterItem filterItem in filteredPlatforms) {
                     platforms.Add(Platforms.GetPlatform(filterItem.Id));
@@ -499,7 +499,7 @@ namespace gaseous_server.Classes
                                     if (File.Exists(gameRomItem.Path))
                                     {
                                         Logging.Log(Logging.LogType.Information, "Collections", "Copying ROM: " + gameRomItem.Name);
-                                        File.Copy(gameRomItem.Path, Path.Combine(ZipGamePath, gameRomItem.Name));
+                                        File.Copy(gameRomItem.Path, Path.Combine(ZipGamePath, gameRomItem.Name), true);
                                     }
                                 }
                             }

gaseous-server/Classes/Filters.cs

@@ -7,11 +7,11 @@ namespace gaseous_server.Classes
 {
     public class Filters
     {
-        public static Dictionary<string, object> Filter(Metadata.AgeRatings.AgeGroups.AgeRestrictionGroupings MaximumAgeRestriction, bool IncludeUnrated)
+        public static Dictionary<string, List<FilterItem>> Filter(Metadata.AgeRatings.AgeGroups.AgeRestrictionGroupings MaximumAgeRestriction, bool IncludeUnrated)
         {
             Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 
-            Dictionary<string, object> FilterSet = new Dictionary<string, object>();
+            Dictionary<string, List<FilterItem>> FilterSet = new Dictionary<string, List<FilterItem>>();
 
             // platforms
             List<FilterItem> platforms = new List<FilterItem>();
@@ -24,7 +24,7 @@ namespace gaseous_server.Classes
                 ageRestriction_Generic += " OR view_Games.AgeGroupId IS NULL";
             }
 
-            string sql = "SELECT DISTINCT Platform.Id, Platform.Abbreviation, Platform.AlternativeName, Platform.`Name`, Platform.PlatformLogo, (SELECT COUNT(*) AS GameCount FROM (SELECT DISTINCT Games_Roms.GameId AS ROMGameId, Games_Roms.PlatformId, view_Games.AgeGroupId FROM Games_Roms LEFT JOIN view_Games ON view_Games.Id = Games_Roms.GameId) Game WHERE Game.PlatformId = Platform.Id AND (" + ageRestriction_Platform + ")) AS GameCount FROM Platform LEFT JOIN Relation_Game_Platforms ON Relation_Game_Platforms.PlatformsId = Platform.Id LEFT JOIN view_Games ON view_Games.Id = Relation_Game_Platforms.GameId HAVING GameCount > 0 ORDER BY Platform.`Name`;";
+            string sql = "SELECT Platform.Id, Platform.`Name`, COUNT(view_Games.Id) AS GameCount FROM view_Games JOIN Relation_Game_Platforms ON Relation_Game_Platforms.GameId = view_Games.Id AND (Relation_Game_Platforms.PlatformsId IN (SELECT DISTINCT PlatformId FROM Games_Roms WHERE Games_Roms.GameId = view_Games.Id)) JOIN Platform ON Platform.Id = Relation_Game_Platforms.PlatformsId WHERE (" + ageRestriction_Generic + ") GROUP BY Platform.`Name` ORDER BY Platform.`Name`;";
             
             DataTable dbResponse = db.ExecuteCMD(sql);
 
@@ -81,22 +81,24 @@ namespace gaseous_server.Classes
             FilterSet.Add("themes", themes);
 
             // age groups
-            List<FilterAgeGrouping> agegroupings = new List<FilterAgeGrouping>();
+            List<FilterItem> agegroupings = new List<FilterItem>();
             sql = "SELECT view_Games.Id, view_Games.AgeGroupId, COUNT(view_Games.Id) AS GameCount FROM view_Games WHERE (" + ageRestriction_Generic + ") GROUP BY view_Games.AgeGroupId ORDER BY view_Games.AgeGroupId DESC;";
             dbResponse = db.ExecuteCMD(sql);
 
             foreach (DataRow dr in dbResponse.Rows)
             {
-                FilterAgeGrouping filterAgeGrouping = new FilterAgeGrouping();
+                FilterItem filterAgeGrouping = new FilterItem();
                 if (dr["AgeGroupId"] == DBNull.Value)
                 {
-                    filterAgeGrouping.Id = (long)AgeRatings.AgeGroups.AgeRestrictionGroupings.Unclassified;
+                    filterAgeGrouping.Id = (int)(long)AgeRatings.AgeGroups.AgeRestrictionGroupings.Unclassified;
-                    filterAgeGrouping.AgeGroup = AgeRatings.AgeGroups.AgeRestrictionGroupings.Unclassified;
+                    filterAgeGrouping.Name = AgeRatings.AgeGroups.AgeRestrictionGroupings.Unclassified.ToString();
                 }
                 else
                 {
-                    filterAgeGrouping.Id = (long)(AgeRatings.AgeGroups.AgeRestrictionGroupings)dr["AgeGroupId"];
+                    long ageGroupLong = (long)dr["AgeGroupId"];
-                    filterAgeGrouping.AgeGroup = (AgeRatings.AgeGroups.AgeRestrictionGroupings)dr["AgeGroupId"];
+                    AgeRatings.AgeGroups.AgeRestrictionGroupings ageGroup = (AgeRatings.AgeGroups.AgeRestrictionGroupings)ageGroupLong;
+                    filterAgeGrouping.Id = ageGroupLong;
+                    filterAgeGrouping.Name = ageGroup.ToString();
                 }
                 filterAgeGrouping.GameCount = (int)(long)dr["GameCount"];
                 agegroupings.Add(filterAgeGrouping);
@@ -108,7 +110,7 @@ namespace gaseous_server.Classes
 
         private static DataTable GetGenericFilterItem(Database db, string Name, string AgeRestriction_Generic)
         {
-            string sql = "SELECT DISTINCT <ITEMNAME>.Id, <ITEMNAME>.`Name`, COUNT(view_Games.Id) AS GameCount FROM <ITEMNAME> LEFT JOIN Relation_Game_<ITEMNAME>s ON Relation_Game_<ITEMNAME>s.<ITEMNAME>sId = <ITEMNAME>.Id LEFT JOIN view_Games ON view_Games.Id = Relation_Game_<ITEMNAME>s.GameId WHERE (" + AgeRestriction_Generic + ") GROUP BY <ITEMNAME>.Id ORDER BY <ITEMNAME>.`Name`;";
+            string sql = "SELECT DISTINCT <ITEMNAME>.Id, <ITEMNAME>.`Name`, COUNT(view_Games.Id) AS GameCount FROM <ITEMNAME> LEFT JOIN Relation_Game_<ITEMNAME>s ON Relation_Game_<ITEMNAME>s.<ITEMNAME>sId = <ITEMNAME>.Id LEFT JOIN view_Games ON view_Games.Id = Relation_Game_<ITEMNAME>s.GameId WHERE (" + AgeRestriction_Generic + ") GROUP BY <ITEMNAME>.Id HAVING GameCount > 0 ORDER BY <ITEMNAME>.`Name`;";
             sql = sql.Replace("<ITEMNAME>", Name);
             DataTable dbResponse = db.ExecuteCMD(sql);
 
@@ -117,6 +119,11 @@ namespace gaseous_server.Classes
 
         public class FilterItem
         {
+            public FilterItem()
+            {
+
+            }
+
             public FilterItem(DataRow dr)
             {
                 this.Id = (long)dr["Id"];
@@ -130,22 +137,5 @@ namespace gaseous_server.Classes
 
             public int GameCount { get; set; }
         }
-
-        public class FilterAgeGrouping
-        {
-            public long Id { get; set; }
-
-            public AgeRatings.AgeGroups.AgeRestrictionGroupings AgeGroup { get ; set; }
-
-            public string Name
-            {
-                get
-                {
-                    return this.AgeGroup.ToString();
-                }
-            }
-
-            public int GameCount { get; set; }
-        }
     }
 }

gaseous-server/Classes/ImportGames.cs

@@ -690,27 +690,35 @@ namespace gaseous_server.Classes
                             IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
 
                             IGDB.Models.Game determinedGame = new Game();
-                            if (determinedPlatform == null)
+                            try
                             {
-                                if (library.DefaultPlatformId == 0)
+                                if (determinedPlatform == null)
                                 {
-                                    determinedPlatform = new IGDB.Models.Platform();
+                                    if (library.DefaultPlatformId == 0)
-                                    determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
+                                    {
+                                        determinedPlatform = new IGDB.Models.Platform();
+                                        determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
+                                    }
+                                    else
+                                    {
+                                        determinedPlatform = Platforms.GetPlatform(library.DefaultPlatformId);
+                                        determinedGame = SearchForGame(sig.Game.Name, library.DefaultPlatformId);
+                                    }
                                 }
                                 else
                                 {
-                                    determinedPlatform = Platforms.GetPlatform(library.DefaultPlatformId);
+                                    determinedGame = SearchForGame(sig.Game.Name, (long)determinedPlatform.Id);
-                                    determinedGame = SearchForGame(sig.Game.Name, library.DefaultPlatformId);
                                 }
-                            }
-                            else
-                            {
-                                determinedGame = SearchForGame(sig.Game.Name, (long)determinedPlatform.Id);
-                            }
 
-                            StoreROM(library, hash, determinedGame, determinedPlatform, sig, LibraryFile);
+                                StoreROM(library, hash, determinedGame, determinedPlatform, sig, LibraryFile);
+                            }
+                            catch (Exception ex)
+                            {
+                                Logging.Log(Logging.LogType.Warning, "Library Scan", " An error occurred while matching orphaned file: " + LibraryFile + ". Skipping.", ex);
+                            }
                         }
                     }
+                    StatusCount += 1;
                 }
                 ClearStatus();
 
@@ -719,6 +727,7 @@ namespace gaseous_server.Classes
 
                 // check all roms to see if their local file still exists
                 Logging.Log(Logging.LogType.Information, "Library Scan", "Checking library files exist on disk");
+                StatusCount = 0;
                 if (dtRoms.Rows.Count > 0)
                 {
                     for (var i = 0; i < dtRoms.Rows.Count; i++)
@@ -727,6 +736,7 @@ namespace gaseous_server.Classes
                         string romPath = (string)dtRoms.Rows[i]["Path"];
                         gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType romMetadataSource = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(int)dtRoms.Rows[i]["MetadataSource"];
 
+                        SetStatus(StatusCount, dtRoms.Rows.Count, "Processing file " + romPath);
                         Logging.Log(Logging.LogType.Information, "Library Scan", " Processing ROM at path " + romPath);
 
                         if (File.Exists(romPath))
@@ -750,6 +760,8 @@ namespace gaseous_server.Classes
                             deleteDict.Add("libraryid", library.Id);
                             db.ExecuteCMD(deleteSql, deleteDict);
                         }
+
+                        StatusCount += 1;
                     }
                 }
 

gaseous-server/Classes/Logging.cs

@@ -109,8 +109,25 @@ namespace gaseous_server.Classes
                         callingProcess = "";
                     }
 
+                    string callingUser;
+                    try
+                    {
+                        if (CallContext.GetData("CallingUser").ToString() == null)
+                        {
+                            callingUser = "";
+                        }
+                        else
+                        {
+                            callingUser = CallContext.GetData("CallingUser").ToString();
+                        }
+                    }
+                    catch
+                    {
+                        callingUser = "";
+                    }
+
                     Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-                    string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception, CorrelationId, CallingProcess) VALUES (@EventTime, @EventType, @Process, @Message, @Exception, @correlationid, @callingprocess);";
+                    string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception, CorrelationId, CallingProcess, CallingUser) VALUES (@EventTime, @EventType, @Process, @Message, @Exception, @correlationid, @callingprocess, @callinguser);";
                     Dictionary<string, object> dbDict = new Dictionary<string, object>();
                     dbDict.Add("EventRententionDate", DateTime.UtcNow.AddDays(Config.LoggingConfiguration.LogRetention * -1));
                     dbDict.Add("EventTime", logItem.EventTime);
@@ -120,6 +137,7 @@ namespace gaseous_server.Classes
                     dbDict.Add("Exception", Common.ReturnValueIfNull(logItem.ExceptionValue, "").ToString());
                     dbDict.Add("correlationid", correlationId);
                     dbDict.Add("callingprocess", callingProcess);
+                    dbDict.Add("callinguser", callingUser);
 
                     try
                     {
@@ -238,6 +256,15 @@ namespace gaseous_server.Classes
                 }
             }
 
+            if (model.CallingUser != null)
+            {
+                if (model.CallingUser.Length > 0)
+                {
+                    dbDict.Add("callingUser", model.CallingUser);
+                    whereClauses.Add("CallingUser = @callingUser");
+                }
+            }
+
             // compile WHERE clause
             string whereClause = "";
             if (whereClauses.Count > 0)
@@ -252,7 +279,8 @@ namespace gaseous_server.Classes
                 {
                     whereClause = "WHERE " + whereClause;
                 }
-                sql = "SELECT * FROM ServerLogs " + whereClause + " ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+                
+                sql = "SELECT ServerLogs.Id, ServerLogs.EventTime, ServerLogs.EventType, ServerLogs.`Process`, ServerLogs.Message, ServerLogs.Exception, ServerLogs.CorrelationId, ServerLogs.CallingProcess, Users.Email FROM ServerLogs LEFT JOIN Users ON ServerLogs.CallingUser = Users.Id " + whereClause + " ORDER BY ServerLogs.Id DESC LIMIT @PageSize OFFSET @PageNumber;";
             }
             else
             {
@@ -260,7 +288,8 @@ namespace gaseous_server.Classes
                 {
                     whereClause = "AND " + whereClause;
                 }
-                sql = "SELECT * FROM ServerLogs WHERE Id < @StartIndex " + whereClause + " ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+                
+                sql = "SELECT ServerLogs.Id, ServerLogs.EventTime, ServerLogs.EventType, ServerLogs.`Process`, ServerLogs.Message, ServerLogs.Exception, ServerLogs.CorrelationId, ServerLogs.CallingProcess, Users.Email FROM ServerLogs LEFT JOIN Users ON ServerLogs.CallingUser = Users.Id  WHERE ServerLogs.Id < @StartIndex " + whereClause + " ORDER BY ServerLogs.Id DESC LIMIT @PageSize OFFSET @PageNumber;";
             }
             DataTable dataTable = db.ExecuteCMD(sql, dbDict);
 
@@ -275,8 +304,9 @@ namespace gaseous_server.Classes
                     Process = (string)row["Process"],
                     Message = (string)row["Message"],
                     ExceptionValue = (string)row["Exception"],
-                    CorrelationId = (string)row["CorrelationId"],
+                    CorrelationId = (string)Common.ReturnValueIfNull(row["CorrelationId"], ""),
-                    CallingProcess = (string)row["CallingProcess"]
+                    CallingProcess = (string)Common.ReturnValueIfNull(row["CallingProcess"], ""),
+                    CallingUser = (string)Common.ReturnValueIfNull(row["Email"], "")
                 };
 
                 logs.Add(log);
@@ -301,6 +331,7 @@ namespace gaseous_server.Classes
             public string Process { get; set; } = "";
             public string CorrelationId { get; set; } = "";
             public string? CallingProcess { get; set; } = "";
+            public string? CallingUser { get; set; } = "";
             private string _Message = "";
             public string Message
             {
@@ -327,6 +358,7 @@ namespace gaseous_server.Classes
             public string? SearchText { get; set; }
             public string? CorrelationId { get; set; }
             public string? CallingProcess { get; set; }
+            public string? CallingUser { get; set; }
         }
     }
 }

gaseous-server/Classes/Metadata/Games.cs

@@ -293,9 +293,37 @@ namespace gaseous_server.Classes.Metadata
             var results = await comms.APIComm<Game>(IGDBClient.Endpoints.Games, fieldList, WhereClause);
             var result = results.First();
 
+            // add artificial unknown platform mapping
+            List<long> platformIds = new List<long>();
+            platformIds.Add(0);
+            if (result.Platforms != null)
+            {
+                if (result.Platforms.Ids != null)
+                {
+                    platformIds.AddRange(result.Platforms.Ids.ToList());
+                }
+            }
+            result.Platforms = new IdentitiesOrValues<Platform>(
+                ids: platformIds.ToArray<long>()
+            );
+
             return result;
         }
         
+        public static void AssignAllGamesToPlatformIdZero()
+        {
+            Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "SELECT * FROM Game;";
+            DataTable gamesTable = db.ExecuteCMD(sql);
+            foreach (DataRow gameRow in gamesTable.Rows)
+            {
+                sql = "DELETE FROM Relation_Game_Platforms WHERE PlatformsId = 0 AND GameId = @Id; INSERT INTO Relation_Game_Platforms (GameId, PlatformsId) VALUES (@Id, 0);";
+                Dictionary<string, object> dbDict = new Dictionary<string, object>();
+                dbDict.Add("Id", (long)gameRow["Id"]);
+                db.ExecuteCMD(sql, dbDict);
+            }
+        }
+
         public static Game[] SearchForGame(string SearchString, long PlatformId, SearchType searchType)
         {
             Task<Game[]> games = _SearchForGame(SearchString, PlatformId, searchType);

gaseous-server/Classes/Metadata/Platforms.cs

@@ -168,6 +168,20 @@ namespace gaseous_server.Classes.Metadata
 
             return result;
         }
+
+        public static void AssignAllPlatformsToGameIdZero()
+        {
+            Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "SELECT * FROM Platform;";
+            DataTable platformsTable = db.ExecuteCMD(sql);
+            foreach (DataRow platformRow in platformsTable.Rows)
+            {
+                sql = "DELETE FROM Relation_Game_Platforms WHERE GameId = 0 AND PlatformsId = @Id; INSERT INTO Relation_Game_Platforms (GameId, PlatformsId) VALUES (0, @Id);";
+                Dictionary<string, object> dbDict = new Dictionary<string, object>();
+                dbDict.Add("Id", (long)platformRow["Id"]);
+                db.ExecuteCMD(sql, dbDict);
+            }
+        }
     }
 }
 

gaseous-server/Classes/Metadata/Storage.cs

@@ -16,32 +16,14 @@ namespace gaseous_server.Classes.Metadata
 			Expired
 		}
 
-        //private static Dictionary<string, MemoryCacheObject> ObjectCache = new Dictionary<string, MemoryCacheObject>();
-
 		public static CacheStatus GetCacheStatus(string Endpoint, string Slug)
 		{
-            // CacheClean();
+            return _GetCacheStatus(Endpoint, "slug", Slug);
-            // if (ObjectCache.ContainsKey(Endpoint + Slug))
-            // {
-            //     return CacheStatus.Current;
-            // }
-            // else
-            // {
-			    return _GetCacheStatus(Endpoint, "slug", Slug);
-            // }
 		}
 
         public static CacheStatus GetCacheStatus(string Endpoint, long Id)
         {
-            // CacheClean();
+            return _GetCacheStatus(Endpoint, "id", Id);
-            // if (ObjectCache.ContainsKey(Endpoint + Id))
-            // {
-            //     return CacheStatus.Current;
-            // }
-            // else
-			// {
-                return _GetCacheStatus(Endpoint, "id", Id);
-            // }
         }
 
         public static CacheStatus GetCacheStatus(DataRow Row)
@@ -185,21 +167,6 @@ namespace gaseous_server.Classes.Metadata
 		{
             string Endpoint = EndpointType.GetType().Name;
 
-            // if (ObjectCache.ContainsKey(Endpoint + SearchValue))
-            // {
-            //     MemoryCacheObject cacheObject = ObjectCache[Endpoint + SearchValue];
-            //     if (cacheObject.ExpiryTime < DateTime.UtcNow)
-            //     {
-            //         // object has expired, remove it
-            //         ObjectCache.Remove(Endpoint + SearchValue);
-            //     }
-            //     else
-            //     {
-            //         // object is valid, return it
-            //         return (T)cacheObject.Object;
-            //     }
-            // }
-
             Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 
             string sql = "SELECT * FROM " + Endpoint + " WHERE " + SearchField + " = @" + SearchField;
@@ -218,19 +185,7 @@ namespace gaseous_server.Classes.Metadata
             {
 				DataRow dataRow = dt.Rows[0];
                 object returnObject = BuildCacheObject<T>(EndpointType, dataRow);
-                // try {
+                
-                //     if (!ObjectCache.ContainsKey(Endpoint + SearchValue))
-                //     {
-                //         ObjectCache.Add(Endpoint + SearchValue, new MemoryCacheObject{
-                //             Object = returnObject
-                //         });
-                //     }
-                // }
-                // catch
-                // {
-                //     // unable add item to cache
-                //     ObjectCache.Clear();
-                // }
                 return (T)returnObject;
             }
         }
@@ -470,29 +425,6 @@ namespace gaseous_server.Classes.Metadata
             }
         }
 
-        // private static void CacheClean()
-        // {
-        //     try
-        //     {
-        //         if (ObjectCache == null)
-        //         {
-        //             ObjectCache = new Dictionary<string, MemoryCacheObject>();
-        //         }
-        //         Dictionary<string, MemoryCacheObject> workCache = ObjectCache;
-        //         foreach (KeyValuePair<string, MemoryCacheObject> objectCache in workCache)
-        //         {
-        //             if (objectCache.Value.ExpiryTime < DateTime.UtcNow)
-        //             {
-        //                 ObjectCache.Remove(objectCache.Key);
-        //             }
-        //         }
-        //     }
-        //     catch
-        //     {
-        //         ObjectCache = new Dictionary<string, MemoryCacheObject>();
-        //     }
-        // }
-
         private class MemoryCacheObject
         {
             public object Object { get; set; }

gaseous-server/Controllers/V1.1/FirstSetupController.cs

@@ -52,15 +52,22 @@ namespace gaseous_server.Controllers
                         NormalizedEmail = model.Email.ToUpper(),
                         SecurityProfile = new SecurityProfileViewModel()
                     };
+                    Logging.Log(Logging.LogType.Information, "First Run", "Creating new account " + model.Email);
                     var result = await _userManager.CreateAsync(user, model.Password);
                     if (result.Succeeded)
                     {
+                        Logging.Log(Logging.LogType.Information, "First Run", "Creation of " + model.Email + " successful.");
+                        Logging.Log(Logging.LogType.Information, "First Run", "Adding Player role to " + model.Email);
                         await _userManager.AddToRoleAsync(user, "Player");
+                        Logging.Log(Logging.LogType.Information, "First Run", "Adding Gamer role to " + model.Email);
                         await _userManager.AddToRoleAsync(user, "Gamer");
+                        Logging.Log(Logging.LogType.Information, "First Run", "Adding Admin role to " + model.Email);
                         await _userManager.AddToRoleAsync(user, "Admin");
 
+                        Logging.Log(Logging.LogType.Information, "First Run", "Signing in as " + model.Email);
                         await _signInManager.SignInAsync(user, isPersistent: true);
                         
+                        Logging.Log(Logging.LogType.Information, "First Run", "Setting first run state to 1");
                         Config.SetSetting("FirstRunStatus", "1");
 
                         return Ok(result);

gaseous-server/Controllers/V1.1/GamesController.cs

@@ -265,7 +265,7 @@ namespace gaseous_server.Controllers.v1_1
 
             if (model.Platform.Count > 0)
             {
-                tempVal = "Games_Roms.PlatformId IN (";
+                tempVal = "Relation_Game_Platforms.PlatformsId IN (";
                 for (int i = 0; i < model.Platform.Count; i++)
                 {
                     if (i > 0)
@@ -439,7 +439,7 @@ namespace gaseous_server.Controllers.v1_1
             string orderByClause = "ORDER BY `" + orderByField + "` " + orderByOrder;
 
             Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-            string sql = "SELECT DISTINCT view_Games.* FROM view_Games LEFT JOIN Games_Roms ON view_Games.Id = Games_Roms.GameId LEFT JOIN Relation_Game_Genres ON view_Games.Id = Relation_Game_Genres.GameId LEFT JOIN Relation_Game_GameModes ON view_Games.Id = Relation_Game_GameModes.GameId LEFT JOIN Relation_Game_PlayerPerspectives ON view_Games.Id = Relation_Game_PlayerPerspectives.GameId LEFT JOIN Relation_Game_Themes ON view_Games.Id = Relation_Game_Themes.GameId " + whereClause + " " + havingClause + " " + orderByClause;
+            string sql = "SELECT DISTINCT view_Games.* FROM view_Games LEFT JOIN Relation_Game_Platforms ON view_Games.Id = Relation_Game_Platforms.GameId AND (Relation_Game_Platforms.PlatformsId IN (SELECT DISTINCT PlatformId FROM Games_Roms WHERE Games_Roms.GameId = view_Games.Id)) LEFT JOIN Relation_Game_Genres ON view_Games.Id = Relation_Game_Genres.GameId LEFT JOIN Relation_Game_GameModes ON view_Games.Id = Relation_Game_GameModes.GameId LEFT JOIN Relation_Game_PlayerPerspectives ON view_Games.Id = Relation_Game_PlayerPerspectives.GameId LEFT JOIN Relation_Game_Themes ON view_Games.Id = Relation_Game_Themes.GameId " + whereClause + " " + havingClause + " " + orderByClause;
 
             List<IGDB.Models.Game> RetVal = new List<IGDB.Models.Game>();
 

gaseous-server/Models/PlatformMapping.cs

@@ -157,18 +157,18 @@ namespace gaseous_server.Models
             if (Update == false)
             {
                 // insert
-                sql = "INSERT INTO PlatformMap (Id, RetroPieDirectoryName, WebEmulator_Type, WebEmulator_Core, AvailableWebEmulators) VALUES (@Id, @RetroPieDirectoryName, @WebEmulator_Type, @WebEmulator_Core, @AvailableWebEmulators)";
+                sql = "INSERT INTO PlatformMap (Id, RetroPieDirectoryName, WebEmulator_Type, WebEmulator_Core, AvailableWebEmulators) VALUES (@Id, @RetroPieDirectoryName, @WebEmulator_Type, @WebEmulator_Core, @AvailableWebEmulators);";
             }
             else
             {
                 // update
                 if (AllowAvailableEmulatorOverwrite == true)
                 {
-                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core, AvailableWebEmulators=@AvailableWebEmulators WHERE Id = @Id";
+                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core, AvailableWebEmulators=@AvailableWebEmulators WHERE Id = @Id; ";
                 }
                 else
                 {
-                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core WHERE Id = @Id";
+                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core WHERE Id = @Id;";
                 }
             }
             dbDict.Add("Id", item.IGDBId);

gaseous-server/ProcessQueue.cs

@@ -148,6 +148,7 @@ namespace gaseous_server
                                         CallingQueueItem = this
                                     };
 
+                                    // clean up
                                     Classes.ImportGame.DeleteOrphanedDirectories(Config.LibraryConfiguration.LibraryImportDirectory);
 
                                     _SaveLastRunTime = true;

gaseous-server/Program.cs

@@ -297,20 +297,6 @@ using (var scope = app.Services.CreateScope())
     }
 }
 
-
-		
-
-app.Use(async (context, next) =>
-{
-    // set the correlation id
-    string correlationId = Guid.NewGuid().ToString();
-    CallContext.SetData("CorrelationId", correlationId);
-    CallContext.SetData("CallingProcess", context.Request.Method + ": " + context.Request.Path);
-
-    context.Response.Headers.Add("x-correlation-id", correlationId.ToString());
-    await next();
-});
-
 app.UseAuthorization();
 
 app.UseDefaultFiles();
@@ -322,6 +308,28 @@ app.UseStaticFiles(new StaticFileOptions
 
 app.MapControllers();
 
+app.Use(async (context, next) =>
+{
+    // set the correlation id
+    string correlationId = Guid.NewGuid().ToString();
+    CallContext.SetData("CorrelationId", correlationId);
+    CallContext.SetData("CallingProcess", context.Request.Method + ": " + context.Request.Path);
+    
+    string userIdentity;
+    try
+    {
+        userIdentity = context.User.Claims.Where(x=>x.Type==System.Security.Claims.ClaimTypes.NameIdentifier).FirstOrDefault().Value;
+    }
+    catch
+    {
+        userIdentity = "";
+    }
+    CallContext.SetData("CallingUser", userIdentity);
+
+    context.Response.Headers.Add("x-correlation-id", correlationId.ToString());
+    await next();
+});
+
 // emergency password recovery if environment variable is set
 // process:
 // - set the environment variable "recoveraccount" to the email address of the account to be recovered
@@ -393,7 +401,9 @@ Config.LibraryConfiguration.InitLibrary();
 
 // insert unknown platform and game if not present
 gaseous_server.Classes.Metadata.Games.GetGame(0, false, false, false);
+gaseous_server.Classes.Metadata.Games.AssignAllGamesToPlatformIdZero();
 gaseous_server.Classes.Metadata.Platforms.GetPlatform(0);
+gaseous_server.Classes.Metadata.Platforms.AssignAllPlatformsToGameIdZero();
 
 // extract platform map if not present
 PlatformMapping.ExtractPlatformMap();

gaseous-server/Support/Database/MySQL/gaseous-1009.sql

@@ -0,0 +1,4 @@
+ALTER TABLE `Games_Roms` 
+ADD INDEX `id_LibraryId` (`LibraryId` ASC) VISIBLE,
+ADD INDEX `id_MD5` USING BTREE (`MD5`) VISIBLE;
+

gaseous-server/Support/Database/MySQL/gaseous-1010.sql

@@ -0,0 +1,23 @@
+CREATE OR REPLACE VIEW `view_Games` AS
+SELECT 
+    a.*, b.AgeGroupId
+FROM
+    view_GamesWithRoms a
+        INNER JOIN
+    (SELECT 
+        view_GamesWithRoms.Id,
+            MAX((SELECT 
+                    AgeGroupId
+                FROM
+                    ClassificationMap
+                WHERE
+                    RatingId = AgeRating.Rating)) AgeGroupId
+    FROM
+        view_GamesWithRoms
+    LEFT JOIN Relation_Game_AgeRatings ON view_GamesWithRoms.Id = Relation_Game_AgeRatings.GameId
+    LEFT JOIN AgeRating ON Relation_Game_AgeRatings.AgeRatingsId = AgeRating.Id
+    GROUP BY Id) b ON a.Id = b.Id
+ORDER BY NameThe;
+
+ALTER TABLE `ServerLogs` 
+ADD COLUMN `CallingUser` VARCHAR(255) NULL AFTER `CallingProcess`;

gaseous-server/gaseous-server.csproj

@@ -49,6 +49,8 @@
     <None Remove="Support\Database\MySQL\gaseous-1006.sql" />
     <None Remove="Support\Database\MySQL\gaseous-1007.sql" />
     <None Remove="Support\Database\MySQL\gaseous-1008.sql" />
+    <None Remove="Support\Database\MySQL\gaseous-1009.sql" />
+    <None Remove="Support\Database\MySQL\gaseous-1010.sql" />
     <None Remove="Classes\Metadata\" />
     <None Remove="Assets\" />
     <None Remove="Assets\Ratings\" />
@@ -180,5 +182,7 @@
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1006.sql" />
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1007.sql" />
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1008.sql" />
+    <EmbeddedResource Include="Support\Database\MySQL\gaseous-1009.sql" />
+    <EmbeddedResource Include="Support\Database\MySQL\gaseous-1010.sql" />
   </ItemGroup>
 </Project>

gaseous-server/wwwroot/pages/settings/logs.html

@@ -139,18 +139,25 @@
                 for (var i = 0; i < result.length; i++) {
                     lastStartIndex = result[i].id;
 
+                    console.log(result[i]);
+
+                    var surroundingRow = document.createElement('tbody');
+                    surroundingRow.setAttribute('colspan', 4);
+                    surroundingRow.className = 'logs_table_row_' + result[i].eventType;
+
                     var newRow = [
-                        //result[i].id,
                         moment(result[i].eventTime).format("YYYY-MM-DD h:mm:ss a"),
                         result[i].eventType,
                         result[i].process,
                         result[i].message
                     ];
 
-                    newTable.appendChild(createTableRow(false, newRow, 'logs_table_row_' + result[i].eventType, 'romcell logs_table_cell'));
+                    surroundingRow.appendChild(createTableRow(false, newRow, '', 'romcell logs_table_cell'));
 
+                    // exception
+                    var exceptionString = '';
                     if (result[i].exceptionValue) {
-                        var exceptionString = "<h3>Exception</h3><pre class='logs_table_exception' style='width: 795px; word-wrap: break-word; overflow-wrap: break-word; overflow-y: scroll;'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br />    ") + "</pre>";
+                        exceptionString = "<strong>Exception</strong><pre class='logs_table_exception' style='width: 795px; word-wrap: break-word; overflow-wrap: break-word; overflow-y: scroll;'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br />    ") + "</pre>";
                         var exRow = document.createElement('tr');
                         var leadCell = document.createElement('td');
                         exRow.appendChild(leadCell);
@@ -158,8 +165,59 @@
                         exCell.colSpan = '3';
                         exCell.innerHTML = exceptionString;
                         exRow.appendChild(exCell);
-                        newTable.appendChild(exRow);
+                        surroundingRow.appendChild(exRow);
                     }
+                    
+                    // calling process
+                    var infoRow = document.createElement('tr');
+                    
+                    var infoRowEmptyCell = document.createElement('td');
+                    infoRowEmptyCell.className = 'romcell';
+
+                    var infoRowDataCell = document.createElement('td');
+                    infoRowDataCell.className = 'romcell';
+                    infoRowDataCell.setAttribute('colspan', 3);
+                    infoRowDataCell.innerHTML = '<strong>Calling process:</strong> ' + result[i].callingProcess;
+
+                    infoRow.appendChild(infoRowEmptyCell);
+                    infoRow.appendChild(infoRowDataCell);
+                    surroundingRow.appendChild(infoRow);
+                    
+                    // initiated by user
+                    if (result[i].callingUser) {
+                        if (result[i].callingUser.length > 0) {
+                            var infoRow3 = document.createElement('tr');
+                            
+                            var infoRowEmptyCell3 = document.createElement('td');
+                            infoRowEmptyCell3.className = 'romcell';
+
+                            var infoRowDataCell3 = document.createElement('td');
+                            infoRowDataCell3.className = 'romcell';
+                            infoRowDataCell3.setAttribute('colspan', 3);
+                            infoRowDataCell3.innerHTML = '<strong>User:</strong> ' + result[i].callingUser + "</a>";
+
+                            infoRow3.appendChild(infoRowEmptyCell3);
+                            infoRow3.appendChild(infoRowDataCell3);
+                            surroundingRow.appendChild(infoRow3);
+                        }
+                    }
+
+                    // correlation id
+                    var infoRow2 = document.createElement('tr');
+                    
+                    var infoRowEmptyCell2 = document.createElement('td');
+                    infoRowEmptyCell2.className = 'romcell';
+
+                    var infoRowDataCell2 = document.createElement('td');
+                    infoRowDataCell2.className = 'romcell';
+                    infoRowDataCell2.setAttribute('colspan', 3);
+                    infoRowDataCell2.innerHTML = '<strong>Correlation Id:</strong> <a class="romlink" href="/index.html?page=settings&sub=logs&correlationid=' + result[i].correlationId + '">' + result[i].correlationId + "</a>";
+
+                    infoRow2.appendChild(infoRowEmptyCell2);
+                    infoRow2.appendChild(infoRowDataCell2);
+                    surroundingRow.appendChild(infoRow2);
+
+                    newTable.appendChild(surroundingRow);
                 }
             },
             function (error) {