Overhaul of SQL queries to (hopefully) improve performance with large libraries (#233)

* Latest round of performance updates

* Improved first set up logging

* Updated logging display

gaseous-server/Classes/Collections.cs

@@ -220,7 +220,7 @@ namespace gaseous_server.Classes
                 }
             } else {
                 // get all platforms to pull from
-                Dictionary<string, object> FilterDict = Filters.Filter(AgeRatings.AgeGroups.AgeRestrictionGroupings.Adult, true);
+                Dictionary<string, List<Filters.FilterItem>> FilterDict = Filters.Filter(AgeRatings.AgeGroups.AgeRestrictionGroupings.Adult, true);
                 List<Classes.Filters.FilterItem> filteredPlatforms = (List<Classes.Filters.FilterItem>)FilterDict["platforms"];
                 foreach (Filters.FilterItem filterItem in filteredPlatforms) {
                     platforms.Add(Platforms.GetPlatform(filterItem.Id));
@@ -499,7 +499,7 @@ namespace gaseous_server.Classes
                                     if (File.Exists(gameRomItem.Path))
                                     {
                                         Logging.Log(Logging.LogType.Information, "Collections", "Copying ROM: " + gameRomItem.Name);
-                                        File.Copy(gameRomItem.Path, Path.Combine(ZipGamePath, gameRomItem.Name));
+                                        File.Copy(gameRomItem.Path, Path.Combine(ZipGamePath, gameRomItem.Name), true);
                                     }
                                 }
                             }

gaseous-server/Classes/Filters.cs

@@ -7,11 +7,11 @@ namespace gaseous_server.Classes
 {
     public class Filters
     {
-        public static Dictionary<string, object> Filter(Metadata.AgeRatings.AgeGroups.AgeRestrictionGroupings MaximumAgeRestriction, bool IncludeUnrated)
+        public static Dictionary<string, List<FilterItem>> Filter(Metadata.AgeRatings.AgeGroups.AgeRestrictionGroupings MaximumAgeRestriction, bool IncludeUnrated)
         {
             Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 
-            Dictionary<string, object> FilterSet = new Dictionary<string, object>();
+            Dictionary<string, List<FilterItem>> FilterSet = new Dictionary<string, List<FilterItem>>();
 
             // platforms
             List<FilterItem> platforms = new List<FilterItem>();
@@ -81,22 +81,24 @@ namespace gaseous_server.Classes
             FilterSet.Add("themes", themes);
 
             // age groups
-            List<FilterAgeGrouping> agegroupings = new List<FilterAgeGrouping>();
+            List<FilterItem> agegroupings = new List<FilterItem>();
             sql = "SELECT view_Games.Id, view_Games.AgeGroupId, COUNT(view_Games.Id) AS GameCount FROM view_Games WHERE (" + ageRestriction_Generic + ") GROUP BY view_Games.AgeGroupId ORDER BY view_Games.AgeGroupId DESC;";
             dbResponse = db.ExecuteCMD(sql);
 
             foreach (DataRow dr in dbResponse.Rows)
             {
-                FilterAgeGrouping filterAgeGrouping = new FilterAgeGrouping();
+                FilterItem filterAgeGrouping = new FilterItem();
                 if (dr["AgeGroupId"] == DBNull.Value)
                 {
-                    filterAgeGrouping.Id = (long)AgeRatings.AgeGroups.AgeRestrictionGroupings.Unclassified;
-                    filterAgeGrouping.AgeGroup = AgeRatings.AgeGroups.AgeRestrictionGroupings.Unclassified;
+                    filterAgeGrouping.Id = (int)(long)AgeRatings.AgeGroups.AgeRestrictionGroupings.Unclassified;
+                    filterAgeGrouping.Name = AgeRatings.AgeGroups.AgeRestrictionGroupings.Unclassified.ToString();
                 }
                 else
                 {
-                    filterAgeGrouping.Id = (long)(AgeRatings.AgeGroups.AgeRestrictionGroupings)dr["AgeGroupId"];
-                    filterAgeGrouping.AgeGroup = (AgeRatings.AgeGroups.AgeRestrictionGroupings)dr["AgeGroupId"];
+                    long ageGroupLong = (long)dr["AgeGroupId"];
+                    AgeRatings.AgeGroups.AgeRestrictionGroupings ageGroup = (AgeRatings.AgeGroups.AgeRestrictionGroupings)ageGroupLong;
+                    filterAgeGrouping.Id = ageGroupLong;
+                    filterAgeGrouping.Name = ageGroup.ToString();
                 }
                 filterAgeGrouping.GameCount = (int)(long)dr["GameCount"];
                 agegroupings.Add(filterAgeGrouping);
@@ -117,6 +119,11 @@ namespace gaseous_server.Classes
 
         public class FilterItem
         {
+            public FilterItem()
+            {
+
+            }
+
             public FilterItem(DataRow dr)
             {
                 this.Id = (long)dr["Id"];
@@ -130,22 +137,5 @@ namespace gaseous_server.Classes
 
             public int GameCount { get; set; }
         }
-
-        public class FilterAgeGrouping
-        {
-            public long Id { get; set; }
-
-            public AgeRatings.AgeGroups.AgeRestrictionGroupings AgeGroup { get ; set; }
-
-            public string Name
-            {
-                get
-                {
-                    return this.AgeGroup.ToString();
-                }
-            }
-
-            public int GameCount { get; set; }
-        }
     }
 }

gaseous-server/Classes/ImportGames.cs

@@ -690,25 +690,32 @@ namespace gaseous_server.Classes
                             IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
 
                             IGDB.Models.Game determinedGame = new Game();
-                            if (determinedPlatform == null)
+                            try
                             {
-                                if (library.DefaultPlatformId == 0)
+                                if (determinedPlatform == null)
                                 {
-                                    determinedPlatform = new IGDB.Models.Platform();
-                                    determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
+                                    if (library.DefaultPlatformId == 0)
+                                    {
+                                        determinedPlatform = new IGDB.Models.Platform();
+                                        determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
+                                    }
+                                    else
+                                    {
+                                        determinedPlatform = Platforms.GetPlatform(library.DefaultPlatformId);
+                                        determinedGame = SearchForGame(sig.Game.Name, library.DefaultPlatformId);
+                                    }
                                 }
                                 else
                                 {
-                                    determinedPlatform = Platforms.GetPlatform(library.DefaultPlatformId);
-                                    determinedGame = SearchForGame(sig.Game.Name, library.DefaultPlatformId);
+                                    determinedGame = SearchForGame(sig.Game.Name, (long)determinedPlatform.Id);
                                 }
-                            }
-                            else
-                            {
-                                determinedGame = SearchForGame(sig.Game.Name, (long)determinedPlatform.Id);
-                            }
 
-                            StoreROM(library, hash, determinedGame, determinedPlatform, sig, LibraryFile);
+                                StoreROM(library, hash, determinedGame, determinedPlatform, sig, LibraryFile);
+                            }
+                            catch (Exception ex)
+                            {
+                                Logging.Log(Logging.LogType.Warning, "Library Scan", " An error occurred while matching orphaned file: " + LibraryFile + ". Skipping.", ex);
+                            }
                         }
                     }
                     StatusCount += 1;
@@ -720,6 +727,7 @@ namespace gaseous_server.Classes
 
                 // check all roms to see if their local file still exists
                 Logging.Log(Logging.LogType.Information, "Library Scan", "Checking library files exist on disk");
+                StatusCount = 0;
                 if (dtRoms.Rows.Count > 0)
                 {
                     for (var i = 0; i < dtRoms.Rows.Count; i++)
@@ -728,6 +736,7 @@ namespace gaseous_server.Classes
                         string romPath = (string)dtRoms.Rows[i]["Path"];
                         gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType romMetadataSource = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(int)dtRoms.Rows[i]["MetadataSource"];
 
+                        SetStatus(StatusCount, dtRoms.Rows.Count, "Processing file " + romPath);
                         Logging.Log(Logging.LogType.Information, "Library Scan", " Processing ROM at path " + romPath);
 
                         if (File.Exists(romPath))
@@ -751,6 +760,8 @@ namespace gaseous_server.Classes
                             deleteDict.Add("libraryid", library.Id);
                             db.ExecuteCMD(deleteSql, deleteDict);
                         }
+
+                        StatusCount += 1;
                     }
                 }
 

gaseous-server/Classes/Logging.cs

@@ -109,8 +109,25 @@ namespace gaseous_server.Classes
                         callingProcess = "";
                     }
 
+                    string callingUser;
+                    try
+                    {
+                        if (CallContext.GetData("CallingUser").ToString() == null)
+                        {
+                            callingUser = "";
+                        }
+                        else
+                        {
+                            callingUser = CallContext.GetData("CallingUser").ToString();
+                        }
+                    }
+                    catch
+                    {
+                        callingUser = "";
+                    }
+
                     Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-                    string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception, CorrelationId, CallingProcess) VALUES (@EventTime, @EventType, @Process, @Message, @Exception, @correlationid, @callingprocess);";
+                    string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception, CorrelationId, CallingProcess, CallingUser) VALUES (@EventTime, @EventType, @Process, @Message, @Exception, @correlationid, @callingprocess, @callinguser);";
                     Dictionary<string, object> dbDict = new Dictionary<string, object>();
                     dbDict.Add("EventRententionDate", DateTime.UtcNow.AddDays(Config.LoggingConfiguration.LogRetention * -1));
                     dbDict.Add("EventTime", logItem.EventTime);
@@ -120,6 +137,7 @@ namespace gaseous_server.Classes
                     dbDict.Add("Exception", Common.ReturnValueIfNull(logItem.ExceptionValue, "").ToString());
                     dbDict.Add("correlationid", correlationId);
                     dbDict.Add("callingprocess", callingProcess);
+                    dbDict.Add("callinguser", callingUser);
 
                     try
                     {
@@ -238,6 +256,15 @@ namespace gaseous_server.Classes
                 }
             }
 
+            if (model.CallingUser != null)
+            {
+                if (model.CallingUser.Length > 0)
+                {
+                    dbDict.Add("callingUser", model.CallingUser);
+                    whereClauses.Add("CallingUser = @callingUser");
+                }
+            }
+
             // compile WHERE clause
             string whereClause = "";
             if (whereClauses.Count > 0)
@@ -252,7 +279,8 @@ namespace gaseous_server.Classes
                 {
                     whereClause = "WHERE " + whereClause;
                 }
-                sql = "SELECT * FROM ServerLogs " + whereClause + " ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+                
+                sql = "SELECT ServerLogs.Id, ServerLogs.EventTime, ServerLogs.EventType, ServerLogs.`Process`, ServerLogs.Message, ServerLogs.Exception, ServerLogs.CorrelationId, ServerLogs.CallingProcess, Users.Email FROM ServerLogs LEFT JOIN Users ON ServerLogs.CallingUser = Users.Id " + whereClause + " ORDER BY ServerLogs.Id DESC LIMIT @PageSize OFFSET @PageNumber;";
             }
             else
             {
@@ -260,7 +288,8 @@ namespace gaseous_server.Classes
                 {
                     whereClause = "AND " + whereClause;
                 }
-                sql = "SELECT * FROM ServerLogs WHERE Id < @StartIndex " + whereClause + " ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+                
+                sql = "SELECT ServerLogs.Id, ServerLogs.EventTime, ServerLogs.EventType, ServerLogs.`Process`, ServerLogs.Message, ServerLogs.Exception, ServerLogs.CorrelationId, ServerLogs.CallingProcess, Users.Email FROM ServerLogs LEFT JOIN Users ON ServerLogs.CallingUser = Users.Id  WHERE ServerLogs.Id < @StartIndex " + whereClause + " ORDER BY ServerLogs.Id DESC LIMIT @PageSize OFFSET @PageNumber;";
             }
             DataTable dataTable = db.ExecuteCMD(sql, dbDict);
 
@@ -276,7 +305,8 @@ namespace gaseous_server.Classes
                     Message = (string)row["Message"],
                     ExceptionValue = (string)row["Exception"],
                     CorrelationId = (string)Common.ReturnValueIfNull(row["CorrelationId"], ""),
-                    CallingProcess = (string)Common.ReturnValueIfNull(row["CallingProcess"], "")
+                    CallingProcess = (string)Common.ReturnValueIfNull(row["CallingProcess"], ""),
+                    CallingUser = (string)Common.ReturnValueIfNull(row["Email"], "")
                 };
 
                 logs.Add(log);
@@ -301,6 +331,7 @@ namespace gaseous_server.Classes
             public string Process { get; set; } = "";
             public string CorrelationId { get; set; } = "";
             public string? CallingProcess { get; set; } = "";
+            public string? CallingUser { get; set; } = "";
             private string _Message = "";
             public string Message
             {
@@ -327,6 +358,7 @@ namespace gaseous_server.Classes
             public string? SearchText { get; set; }
             public string? CorrelationId { get; set; }
             public string? CallingProcess { get; set; }
+            public string? CallingUser { get; set; }
         }
     }
 }

gaseous-server/Controllers/V1.1/FirstSetupController.cs

@@ -52,15 +52,22 @@ namespace gaseous_server.Controllers
                         NormalizedEmail = model.Email.ToUpper(),
                         SecurityProfile = new SecurityProfileViewModel()
                     };
+                    Logging.Log(Logging.LogType.Information, "First Run", "Creating new account " + model.Email);
                     var result = await _userManager.CreateAsync(user, model.Password);
                     if (result.Succeeded)
                     {
+                        Logging.Log(Logging.LogType.Information, "First Run", "Creation of " + model.Email + " successful.");
+                        Logging.Log(Logging.LogType.Information, "First Run", "Adding Player role to " + model.Email);
                         await _userManager.AddToRoleAsync(user, "Player");
+                        Logging.Log(Logging.LogType.Information, "First Run", "Adding Gamer role to " + model.Email);
                         await _userManager.AddToRoleAsync(user, "Gamer");
+                        Logging.Log(Logging.LogType.Information, "First Run", "Adding Admin role to " + model.Email);
                         await _userManager.AddToRoleAsync(user, "Admin");
 
+                        Logging.Log(Logging.LogType.Information, "First Run", "Signing in as " + model.Email);
                         await _signInManager.SignInAsync(user, isPersistent: true);
                         
+                        Logging.Log(Logging.LogType.Information, "First Run", "Setting first run state to 1");
                         Config.SetSetting("FirstRunStatus", "1");
 
                         return Ok(result);

gaseous-server/ProcessQueue.cs

@@ -148,6 +148,7 @@ namespace gaseous_server
                                         CallingQueueItem = this
                                     };
 
+                                    // clean up
                                     Classes.ImportGame.DeleteOrphanedDirectories(Config.LibraryConfiguration.LibraryImportDirectory);
 
                                     _SaveLastRunTime = true;

gaseous-server/Program.cs

@@ -297,20 +297,6 @@ using (var scope = app.Services.CreateScope())
     }
 }
 
-
-		
-
-app.Use(async (context, next) =>
-{
-    // set the correlation id
-    string correlationId = Guid.NewGuid().ToString();
-    CallContext.SetData("CorrelationId", correlationId);
-    CallContext.SetData("CallingProcess", context.Request.Method + ": " + context.Request.Path);
-
-    context.Response.Headers.Add("x-correlation-id", correlationId.ToString());
-    await next();
-});
-
 app.UseAuthorization();
 
 app.UseDefaultFiles();
@@ -322,6 +308,28 @@ app.UseStaticFiles(new StaticFileOptions
 
 app.MapControllers();
 
+app.Use(async (context, next) =>
+{
+    // set the correlation id
+    string correlationId = Guid.NewGuid().ToString();
+    CallContext.SetData("CorrelationId", correlationId);
+    CallContext.SetData("CallingProcess", context.Request.Method + ": " + context.Request.Path);
+    
+    string userIdentity;
+    try
+    {
+        userIdentity = context.User.Claims.Where(x=>x.Type==System.Security.Claims.ClaimTypes.NameIdentifier).FirstOrDefault().Value;
+    }
+    catch
+    {
+        userIdentity = "";
+    }
+    CallContext.SetData("CallingUser", userIdentity);
+
+    context.Response.Headers.Add("x-correlation-id", correlationId.ToString());
+    await next();
+});
+
 // emergency password recovery if environment variable is set
 // process:
 // - set the environment variable "recoveraccount" to the email address of the account to be recovered

gaseous-server/Support/Database/MySQL/gaseous-1010.sql

@@ -0,0 +1,23 @@
+CREATE OR REPLACE VIEW `view_Games` AS
+SELECT 
+    a.*, b.AgeGroupId
+FROM
+    view_GamesWithRoms a
+        INNER JOIN
+    (SELECT 
+        view_GamesWithRoms.Id,
+            MAX((SELECT 
+                    AgeGroupId
+                FROM
+                    ClassificationMap
+                WHERE
+                    RatingId = AgeRating.Rating)) AgeGroupId
+    FROM
+        view_GamesWithRoms
+    LEFT JOIN Relation_Game_AgeRatings ON view_GamesWithRoms.Id = Relation_Game_AgeRatings.GameId
+    LEFT JOIN AgeRating ON Relation_Game_AgeRatings.AgeRatingsId = AgeRating.Id
+    GROUP BY Id) b ON a.Id = b.Id
+ORDER BY NameThe;
+
+ALTER TABLE `ServerLogs` 
+ADD COLUMN `CallingUser` VARCHAR(255) NULL AFTER `CallingProcess`;

gaseous-server/gaseous-server.csproj

@@ -50,6 +50,7 @@
     <None Remove="Support\Database\MySQL\gaseous-1007.sql" />
     <None Remove="Support\Database\MySQL\gaseous-1008.sql" />
     <None Remove="Support\Database\MySQL\gaseous-1009.sql" />
+    <None Remove="Support\Database\MySQL\gaseous-1010.sql" />
     <None Remove="Classes\Metadata\" />
     <None Remove="Assets\" />
     <None Remove="Assets\Ratings\" />
@@ -182,5 +183,6 @@
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1007.sql" />
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1008.sql" />
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1009.sql" />
+    <EmbeddedResource Include="Support\Database\MySQL\gaseous-1010.sql" />
   </ItemGroup>
 </Project>

gaseous-server/wwwroot/pages/settings/logs.html

@@ -139,18 +139,25 @@
                 for (var i = 0; i < result.length; i++) {
                     lastStartIndex = result[i].id;
 
+                    console.log(result[i]);
+
+                    var surroundingRow = document.createElement('tbody');
+                    surroundingRow.setAttribute('colspan', 4);
+                    surroundingRow.className = 'logs_table_row_' + result[i].eventType;
+
                     var newRow = [
-                        //result[i].id,
                         moment(result[i].eventTime).format("YYYY-MM-DD h:mm:ss a"),
                         result[i].eventType,
                         result[i].process,
                         result[i].message
                     ];
 
-                    newTable.appendChild(createTableRow(false, newRow, 'logs_table_row_' + result[i].eventType, 'romcell logs_table_cell'));
+                    surroundingRow.appendChild(createTableRow(false, newRow, '', 'romcell logs_table_cell'));
 
+                    // exception
+                    var exceptionString = '';
                     if (result[i].exceptionValue) {
-                        var exceptionString = "<h3>Exception</h3><pre class='logs_table_exception' style='width: 795px; word-wrap: break-word; overflow-wrap: break-word; overflow-y: scroll;'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br />    ") + "</pre>";
+                        exceptionString = "<strong>Exception</strong><pre class='logs_table_exception' style='width: 795px; word-wrap: break-word; overflow-wrap: break-word; overflow-y: scroll;'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br />    ") + "</pre>";
                         var exRow = document.createElement('tr');
                         var leadCell = document.createElement('td');
                         exRow.appendChild(leadCell);
@@ -158,8 +165,59 @@
                         exCell.colSpan = '3';
                         exCell.innerHTML = exceptionString;
                         exRow.appendChild(exCell);
-                        newTable.appendChild(exRow);
+                        surroundingRow.appendChild(exRow);
                     }
+                    
+                    // calling process
+                    var infoRow = document.createElement('tr');
+                    
+                    var infoRowEmptyCell = document.createElement('td');
+                    infoRowEmptyCell.className = 'romcell';
+
+                    var infoRowDataCell = document.createElement('td');
+                    infoRowDataCell.className = 'romcell';
+                    infoRowDataCell.setAttribute('colspan', 3);
+                    infoRowDataCell.innerHTML = '<strong>Calling process:</strong> ' + result[i].callingProcess;
+
+                    infoRow.appendChild(infoRowEmptyCell);
+                    infoRow.appendChild(infoRowDataCell);
+                    surroundingRow.appendChild(infoRow);
+                    
+                    // initiated by user
+                    if (result[i].callingUser) {
+                        if (result[i].callingUser.length > 0) {
+                            var infoRow3 = document.createElement('tr');
+                            
+                            var infoRowEmptyCell3 = document.createElement('td');
+                            infoRowEmptyCell3.className = 'romcell';
+
+                            var infoRowDataCell3 = document.createElement('td');
+                            infoRowDataCell3.className = 'romcell';
+                            infoRowDataCell3.setAttribute('colspan', 3);
+                            infoRowDataCell3.innerHTML = '<strong>User:</strong> ' + result[i].callingUser + "</a>";
+
+                            infoRow3.appendChild(infoRowEmptyCell3);
+                            infoRow3.appendChild(infoRowDataCell3);
+                            surroundingRow.appendChild(infoRow3);
+                        }
+                    }
+
+                    // correlation id
+                    var infoRow2 = document.createElement('tr');
+                    
+                    var infoRowEmptyCell2 = document.createElement('td');
+                    infoRowEmptyCell2.className = 'romcell';
+
+                    var infoRowDataCell2 = document.createElement('td');
+                    infoRowDataCell2.className = 'romcell';
+                    infoRowDataCell2.setAttribute('colspan', 3);
+                    infoRowDataCell2.innerHTML = '<strong>Correlation Id:</strong> <a class="romlink" href="/index.html?page=settings&sub=logs&correlationid=' + result[i].correlationId + '">' + result[i].correlationId + "</a>";
+
+                    infoRow2.appendChild(infoRowEmptyCell2);
+                    infoRow2.appendChild(infoRowDataCell2);
+                    surroundingRow.appendChild(infoRow2);
+
+                    newTable.appendChild(surroundingRow);
                 }
             },
             function (error) {