Many queries re-written to improve performance (#232)

gaseous-server/Classes/Filters.cs

@@ -24,7 +24,7 @@ namespace gaseous_server.Classes
                 ageRestriction_Generic += " OR view_Games.AgeGroupId IS NULL";
             }
 
-            string sql = "SELECT DISTINCT Platform.Id, Platform.Abbreviation, Platform.AlternativeName, Platform.`Name`, Platform.PlatformLogo, (SELECT COUNT(*) AS GameCount FROM (SELECT DISTINCT Games_Roms.GameId AS ROMGameId, Games_Roms.PlatformId, view_Games.AgeGroupId FROM Games_Roms LEFT JOIN view_Games ON view_Games.Id = Games_Roms.GameId) Game WHERE Game.PlatformId = Platform.Id AND (" + ageRestriction_Platform + ")) AS GameCount FROM Platform LEFT JOIN Relation_Game_Platforms ON Relation_Game_Platforms.PlatformsId = Platform.Id LEFT JOIN view_Games ON view_Games.Id = Relation_Game_Platforms.GameId HAVING GameCount > 0 ORDER BY Platform.`Name`;";
+            string sql = "SELECT Platform.Id, Platform.`Name`, COUNT(view_Games.Id) AS GameCount FROM view_Games JOIN Relation_Game_Platforms ON Relation_Game_Platforms.GameId = view_Games.Id AND (Relation_Game_Platforms.PlatformsId IN (SELECT DISTINCT PlatformId FROM Games_Roms WHERE Games_Roms.GameId = view_Games.Id)) JOIN Platform ON Platform.Id = Relation_Game_Platforms.PlatformsId WHERE (" + ageRestriction_Generic + ") GROUP BY Platform.`Name` ORDER BY Platform.`Name`;";
             
             DataTable dbResponse = db.ExecuteCMD(sql);
 
@@ -108,7 +108,7 @@ namespace gaseous_server.Classes
 
         private static DataTable GetGenericFilterItem(Database db, string Name, string AgeRestriction_Generic)
         {
-            string sql = "SELECT DISTINCT <ITEMNAME>.Id, <ITEMNAME>.`Name`, COUNT(view_Games.Id) AS GameCount FROM <ITEMNAME> LEFT JOIN Relation_Game_<ITEMNAME>s ON Relation_Game_<ITEMNAME>s.<ITEMNAME>sId = <ITEMNAME>.Id LEFT JOIN view_Games ON view_Games.Id = Relation_Game_<ITEMNAME>s.GameId WHERE (" + AgeRestriction_Generic + ") GROUP BY <ITEMNAME>.Id ORDER BY <ITEMNAME>.`Name`;";
+            string sql = "SELECT DISTINCT <ITEMNAME>.Id, <ITEMNAME>.`Name`, COUNT(view_Games.Id) AS GameCount FROM <ITEMNAME> LEFT JOIN Relation_Game_<ITEMNAME>s ON Relation_Game_<ITEMNAME>s.<ITEMNAME>sId = <ITEMNAME>.Id LEFT JOIN view_Games ON view_Games.Id = Relation_Game_<ITEMNAME>s.GameId WHERE (" + AgeRestriction_Generic + ") GROUP BY <ITEMNAME>.Id HAVING GameCount > 0 ORDER BY <ITEMNAME>.`Name`;";
             sql = sql.Replace("<ITEMNAME>", Name);
             DataTable dbResponse = db.ExecuteCMD(sql);
 

gaseous-server/Classes/ImportGames.cs

@@ -711,6 +711,7 @@ namespace gaseous_server.Classes
                             StoreROM(library, hash, determinedGame, determinedPlatform, sig, LibraryFile);
                         }
                     }
+                    StatusCount += 1;
                 }
                 ClearStatus();
 

gaseous-server/Classes/Logging.cs

@@ -275,8 +275,8 @@ namespace gaseous_server.Classes
                     Process = (string)row["Process"],
                     Message = (string)row["Message"],
                     ExceptionValue = (string)row["Exception"],
-                    CorrelationId = (string)row["CorrelationId"],
-                    CallingProcess = (string)row["CallingProcess"]
+                    CorrelationId = (string)Common.ReturnValueIfNull(row["CorrelationId"], ""),
+                    CallingProcess = (string)Common.ReturnValueIfNull(row["CallingProcess"], "")
                 };
 
                 logs.Add(log);

gaseous-server/Classes/Metadata/Games.cs

@@ -293,9 +293,37 @@ namespace gaseous_server.Classes.Metadata
             var results = await comms.APIComm<Game>(IGDBClient.Endpoints.Games, fieldList, WhereClause);
             var result = results.First();
 
+            // add artificial unknown platform mapping
+            List<long> platformIds = new List<long>();
+            platformIds.Add(0);
+            if (result.Platforms != null)
+            {
+                if (result.Platforms.Ids != null)
+                {
+                    platformIds.AddRange(result.Platforms.Ids.ToList());
+                }
+            }
+            result.Platforms = new IdentitiesOrValues<Platform>(
+                ids: platformIds.ToArray<long>()
+            );
+
             return result;
         }
         
+        public static void AssignAllGamesToPlatformIdZero()
+        {
+            Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "SELECT * FROM Game;";
+            DataTable gamesTable = db.ExecuteCMD(sql);
+            foreach (DataRow gameRow in gamesTable.Rows)
+            {
+                sql = "DELETE FROM Relation_Game_Platforms WHERE PlatformsId = 0 AND GameId = @Id; INSERT INTO Relation_Game_Platforms (GameId, PlatformsId) VALUES (@Id, 0);";
+                Dictionary<string, object> dbDict = new Dictionary<string, object>();
+                dbDict.Add("Id", (long)gameRow["Id"]);
+                db.ExecuteCMD(sql, dbDict);
+            }
+        }
+
         public static Game[] SearchForGame(string SearchString, long PlatformId, SearchType searchType)
         {
             Task<Game[]> games = _SearchForGame(SearchString, PlatformId, searchType);

gaseous-server/Classes/Metadata/Platforms.cs

@@ -168,6 +168,20 @@ namespace gaseous_server.Classes.Metadata
 
             return result;
         }
+
+        public static void AssignAllPlatformsToGameIdZero()
+        {
+            Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "SELECT * FROM Platform;";
+            DataTable platformsTable = db.ExecuteCMD(sql);
+            foreach (DataRow platformRow in platformsTable.Rows)
+            {
+                sql = "DELETE FROM Relation_Game_Platforms WHERE GameId = 0 AND PlatformsId = @Id; INSERT INTO Relation_Game_Platforms (GameId, PlatformsId) VALUES (0, @Id);";
+                Dictionary<string, object> dbDict = new Dictionary<string, object>();
+                dbDict.Add("Id", (long)platformRow["Id"]);
+                db.ExecuteCMD(sql, dbDict);
+            }
+        }
     }
 }
 

gaseous-server/Classes/Metadata/Storage.cs

@@ -16,32 +16,14 @@ namespace gaseous_server.Classes.Metadata
 			Expired
 		}
 
-        //private static Dictionary<string, MemoryCacheObject> ObjectCache = new Dictionary<string, MemoryCacheObject>();
-
 		public static CacheStatus GetCacheStatus(string Endpoint, string Slug)
 		{
-            // CacheClean();
-            // if (ObjectCache.ContainsKey(Endpoint + Slug))
-            // {
-            //     return CacheStatus.Current;
-            // }
-            // else
-            // {
             return _GetCacheStatus(Endpoint, "slug", Slug);
-            // }
 		}
 
         public static CacheStatus GetCacheStatus(string Endpoint, long Id)
         {
-            // CacheClean();
-            // if (ObjectCache.ContainsKey(Endpoint + Id))
-            // {
-            //     return CacheStatus.Current;
-            // }
-            // else
-			// {
             return _GetCacheStatus(Endpoint, "id", Id);
-            // }
         }
 
         public static CacheStatus GetCacheStatus(DataRow Row)
@@ -185,21 +167,6 @@ namespace gaseous_server.Classes.Metadata
 		{
             string Endpoint = EndpointType.GetType().Name;
 
-            // if (ObjectCache.ContainsKey(Endpoint + SearchValue))
-            // {
-            //     MemoryCacheObject cacheObject = ObjectCache[Endpoint + SearchValue];
-            //     if (cacheObject.ExpiryTime < DateTime.UtcNow)
-            //     {
-            //         // object has expired, remove it
-            //         ObjectCache.Remove(Endpoint + SearchValue);
-            //     }
-            //     else
-            //     {
-            //         // object is valid, return it
-            //         return (T)cacheObject.Object;
-            //     }
-            // }
-
             Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 
             string sql = "SELECT * FROM " + Endpoint + " WHERE " + SearchField + " = @" + SearchField;
@@ -218,19 +185,7 @@ namespace gaseous_server.Classes.Metadata
             {
 				DataRow dataRow = dt.Rows[0];
                 object returnObject = BuildCacheObject<T>(EndpointType, dataRow);
-                // try {
-                //     if (!ObjectCache.ContainsKey(Endpoint + SearchValue))
-                //     {
-                //         ObjectCache.Add(Endpoint + SearchValue, new MemoryCacheObject{
-                //             Object = returnObject
-                //         });
-                //     }
-                // }
-                // catch
-                // {
-                //     // unable add item to cache
-                //     ObjectCache.Clear();
-                // }
+                
                 return (T)returnObject;
             }
         }
@@ -470,29 +425,6 @@ namespace gaseous_server.Classes.Metadata
             }
         }
 
-        // private static void CacheClean()
-        // {
-        //     try
-        //     {
-        //         if (ObjectCache == null)
-        //         {
-        //             ObjectCache = new Dictionary<string, MemoryCacheObject>();
-        //         }
-        //         Dictionary<string, MemoryCacheObject> workCache = ObjectCache;
-        //         foreach (KeyValuePair<string, MemoryCacheObject> objectCache in workCache)
-        //         {
-        //             if (objectCache.Value.ExpiryTime < DateTime.UtcNow)
-        //             {
-        //                 ObjectCache.Remove(objectCache.Key);
-        //             }
-        //         }
-        //     }
-        //     catch
-        //     {
-        //         ObjectCache = new Dictionary<string, MemoryCacheObject>();
-        //     }
-        // }
-
         private class MemoryCacheObject
         {
             public object Object { get; set; }

gaseous-server/Controllers/V1.1/GamesController.cs

@@ -265,7 +265,7 @@ namespace gaseous_server.Controllers.v1_1
 
             if (model.Platform.Count > 0)
             {
-                tempVal = "Games_Roms.PlatformId IN (";
+                tempVal = "Relation_Game_Platforms.PlatformsId IN (";
                 for (int i = 0; i < model.Platform.Count; i++)
                 {
                     if (i > 0)
@@ -439,7 +439,7 @@ namespace gaseous_server.Controllers.v1_1
             string orderByClause = "ORDER BY `" + orderByField + "` " + orderByOrder;
 
             Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-            string sql = "SELECT DISTINCT view_Games.* FROM view_Games LEFT JOIN Games_Roms ON view_Games.Id = Games_Roms.GameId LEFT JOIN Relation_Game_Genres ON view_Games.Id = Relation_Game_Genres.GameId LEFT JOIN Relation_Game_GameModes ON view_Games.Id = Relation_Game_GameModes.GameId LEFT JOIN Relation_Game_PlayerPerspectives ON view_Games.Id = Relation_Game_PlayerPerspectives.GameId LEFT JOIN Relation_Game_Themes ON view_Games.Id = Relation_Game_Themes.GameId " + whereClause + " " + havingClause + " " + orderByClause;
+            string sql = "SELECT DISTINCT view_Games.* FROM view_Games LEFT JOIN Relation_Game_Platforms ON view_Games.Id = Relation_Game_Platforms.GameId AND (Relation_Game_Platforms.PlatformsId IN (SELECT DISTINCT PlatformId FROM Games_Roms WHERE Games_Roms.GameId = view_Games.Id)) LEFT JOIN Relation_Game_Genres ON view_Games.Id = Relation_Game_Genres.GameId LEFT JOIN Relation_Game_GameModes ON view_Games.Id = Relation_Game_GameModes.GameId LEFT JOIN Relation_Game_PlayerPerspectives ON view_Games.Id = Relation_Game_PlayerPerspectives.GameId LEFT JOIN Relation_Game_Themes ON view_Games.Id = Relation_Game_Themes.GameId " + whereClause + " " + havingClause + " " + orderByClause;
 
             List<IGDB.Models.Game> RetVal = new List<IGDB.Models.Game>();
 

gaseous-server/Models/PlatformMapping.cs

@@ -157,18 +157,18 @@ namespace gaseous_server.Models
             if (Update == false)
             {
                 // insert
-                sql = "INSERT INTO PlatformMap (Id, RetroPieDirectoryName, WebEmulator_Type, WebEmulator_Core, AvailableWebEmulators) VALUES (@Id, @RetroPieDirectoryName, @WebEmulator_Type, @WebEmulator_Core, @AvailableWebEmulators)";
+                sql = "INSERT INTO PlatformMap (Id, RetroPieDirectoryName, WebEmulator_Type, WebEmulator_Core, AvailableWebEmulators) VALUES (@Id, @RetroPieDirectoryName, @WebEmulator_Type, @WebEmulator_Core, @AvailableWebEmulators);";
             }
             else
             {
                 // update
                 if (AllowAvailableEmulatorOverwrite == true)
                 {
-                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core, AvailableWebEmulators=@AvailableWebEmulators WHERE Id = @Id";
+                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core, AvailableWebEmulators=@AvailableWebEmulators WHERE Id = @Id; ";
                 }
                 else
                 {
-                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core WHERE Id = @Id";
+                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core WHERE Id = @Id;";
                 }
             }
             dbDict.Add("Id", item.IGDBId);

gaseous-server/Program.cs

@@ -393,7 +393,9 @@ Config.LibraryConfiguration.InitLibrary();
 
 // insert unknown platform and game if not present
 gaseous_server.Classes.Metadata.Games.GetGame(0, false, false, false);
+gaseous_server.Classes.Metadata.Games.AssignAllGamesToPlatformIdZero();
 gaseous_server.Classes.Metadata.Platforms.GetPlatform(0);
+gaseous_server.Classes.Metadata.Platforms.AssignAllPlatformsToGameIdZero();
 
 // extract platform map if not present
 PlatformMapping.ExtractPlatformMap();

gaseous-server/Support/Database/MySQL/gaseous-1009.sql

@@ -0,0 +1,4 @@
+ALTER TABLE `Games_Roms` 
+ADD INDEX `id_LibraryId` (`LibraryId` ASC) VISIBLE,
+ADD INDEX `id_MD5` USING BTREE (`MD5`) VISIBLE;
+

gaseous-server/gaseous-server.csproj

@@ -49,6 +49,7 @@
     <None Remove="Support\Database\MySQL\gaseous-1006.sql" />
     <None Remove="Support\Database\MySQL\gaseous-1007.sql" />
     <None Remove="Support\Database\MySQL\gaseous-1008.sql" />
+    <None Remove="Support\Database\MySQL\gaseous-1009.sql" />
     <None Remove="Classes\Metadata\" />
     <None Remove="Assets\" />
     <None Remove="Assets\Ratings\" />
@@ -180,5 +181,6 @@
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1006.sql" />
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1007.sql" />
     <EmbeddedResource Include="Support\Database\MySQL\gaseous-1008.sql" />
+    <EmbeddedResource Include="Support\Database\MySQL\gaseous-1009.sql" />
   </ItemGroup>
 </Project>