Removed import block from LibraryScan (#150)

* Updated readme with database limitations

* Wrapped all IGDB calls (except search) in try catch blocks

.github/workflows/BuildDockerOnTag-Prerelease.yml

@@ -0,0 +1,38 @@
+name: Build Pre-release Docker Image
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-preview.[0-9]'
+      - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+      - name: Install dotnet tool
+        run: dotnet tool install -g dotnetCampus.TagToVersion
+      - name: Set tag to version  
+        run: dotnet TagToVersion -t ${{ github.ref }}
+      - name: Sign in to Nuget
+        run: dotnet nuget add source --username michael-j-green --password ${{ secrets.NUGETKEY }} --store-password-in-clear-text --name github "https://nuget.pkg.github.com/gaseous-project/index.json"
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: linux/amd64 #,linux/arm64
+          push: true
+          tags: gaseousgames/gaseousserver:${{ github.ref_name}}

.github/workflows/BuildDockerOnTag-Release.yml

@@ -1,39 +1,34 @@
-name: Build Docker Image on New Tag
+name: Build Release Docker Image
 
 on:
   push:
     tags:
-      - '*'
+      - 'v[0-9]+.[0-9]+.[0-9]+'
 
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v3
         with:
           submodules: 'true'
-      -
-        name: Install dotnet tool
+      - name: Install dotnet tool
         run: dotnet tool install -g dotnetCampus.TagToVersion
-      -
-        name: Set tag to version  
+      - name: Set tag to version  
         run: dotnet TagToVersion -t ${{ github.ref }}
-      -
-        name: Set up QEMU
+      - name: Sign in to Nuget
+        run: dotnet nuget add source --username michael-j-green --password ${{ secrets.NUGETKEY }} --store-password-in-clear-text --name github "https://nuget.pkg.github.com/gaseous-project/index.json"
+      - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
-      -
-        name: Set up Docker Buildx
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      -
-        name: Login to Docker Hub
+      - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      -
-        name: Build and push
+      - name: Build and push
         uses: docker/build-push-action@v4
         with:
           context: .

.github/workflows/codeql.yml

@@ -0,0 +1,85 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main", "branch-v*.*.*" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '21 11 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'csharp', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      
+    - name: Sign in to Nuget
+      run: dotnet nuget add source --username michael-j-green --password ${{ secrets.NUGETKEY }} --store-password-in-clear-text --name github "https://nuget.pkg.github.com/gaseous-project/index.json"
+        
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/dotnet.yml

@@ -0,0 +1,24 @@
+name: .NET
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v1
+        with:
+          dotnet-version: 7.0.x
+      - name: Sign in to Nuget
+        run: dotnet nuget add source --username michael-j-green --password ${{ secrets.NUGETKEY }} --store-password-in-clear-text --name github "https://nuget.pkg.github.com/gaseous-project/index.json"
+      - name: Restore dependencies
+        run: dotnet restore
+      - name: Build
+        run: dotnet build --no-restore

Gaseous.sln

@@ -1,16 +1,8 @@
-
+
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 25.0.1704.4
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gaseous-identifier-testapp", "gaseous-identifier\gaseous-identifier-testapp.csproj", "{F5C42134-5372-430A-A9AE-1871981850DB}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gaseous-signature-parser", "gaseous-signature-parser\gaseous-signature-parser.csproj", "{DAEBBB82-5051-43FD-A406-F9D64A38F468}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gaseous-romsignatureobject", "gaseous-romsignatureobject\gaseous-romsignatureobject.csproj", "{9DCD243D-37CE-4562-8411-B5242B687D4F}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gaseous-signature-ingestor", "gaseous-signature-ingestor\gaseous-signature-ingestor.csproj", "{86DF6E45-2C2B-4C30-AEC1-E7EF8C5CEA7D}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gaseous-tools", "gaseous-tools\gaseous-tools.csproj", "{08FE408A-5EC1-4110-ABD8-D19A1155B8CE}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gaseous-server", "gaseous-server\gaseous-server.csproj", "{A01D2EFF-C82E-473B-84D7-7C25E736F5D2}"
@@ -37,30 +29,14 @@ Global
 		Release|Any CPU = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{F5C42134-5372-430A-A9AE-1871981850DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F5C42134-5372-430A-A9AE-1871981850DB}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F5C42134-5372-430A-A9AE-1871981850DB}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F5C42134-5372-430A-A9AE-1871981850DB}.Release|Any CPU.Build.0 = Release|Any CPU
 		{08699C93-15CD-4E39-9053-D3C8056CE938}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{08699C93-15CD-4E39-9053-D3C8056CE938}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{08699C93-15CD-4E39-9053-D3C8056CE938}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{08699C93-15CD-4E39-9053-D3C8056CE938}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DAEBBB82-5051-43FD-A406-F9D64A38F468}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DAEBBB82-5051-43FD-A406-F9D64A38F468}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DAEBBB82-5051-43FD-A406-F9D64A38F468}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DAEBBB82-5051-43FD-A406-F9D64A38F468}.Release|Any CPU.Build.0 = Release|Any CPU
 		{FFCEC386-033F-4772-A45B-D33579F2E5EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{FFCEC386-033F-4772-A45B-D33579F2E5EE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{FFCEC386-033F-4772-A45B-D33579F2E5EE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{FFCEC386-033F-4772-A45B-D33579F2E5EE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9DCD243D-37CE-4562-8411-B5242B687D4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9DCD243D-37CE-4562-8411-B5242B687D4F}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9DCD243D-37CE-4562-8411-B5242B687D4F}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9DCD243D-37CE-4562-8411-B5242B687D4F}.Release|Any CPU.Build.0 = Release|Any CPU
-		{86DF6E45-2C2B-4C30-AEC1-E7EF8C5CEA7D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{86DF6E45-2C2B-4C30-AEC1-E7EF8C5CEA7D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{86DF6E45-2C2B-4C30-AEC1-E7EF8C5CEA7D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{86DF6E45-2C2B-4C30-AEC1-E7EF8C5CEA7D}.Release|Any CPU.Build.0 = Release|Any CPU
 		{08FE408A-5EC1-4110-ABD8-D19A1155B8CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{08FE408A-5EC1-4110-ABD8-D19A1155B8CE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{08FE408A-5EC1-4110-ABD8-D19A1155B8CE}.Release|Any CPU.ActiveCfg = Release|Any CPU

README.MD

@@ -1,6 +1,14 @@
 # Gaseous Server
 
-This is the server for the Gaseous system. All your games and metadata are stored within.
+This is the server for the Gaseous system. It offers ROM and title management, as well as some basic in browser emulation of those ROM's.
+
+## Warning
+
+This project is currently not suitable for being exposed to the internet.
+1. there is currently no authentication support, meaning anyone could trash your library
+2. the server has not been hardened for exposure to the internet - so there maybe unknown vulnerabilities
+
+If you expose the server to the internet, **you do so at your own risk**.
 
 ## Screenshots
 ![Library](./screenshots/Library.png)
@@ -8,9 +16,11 @@ This is the server for the Gaseous system. All your games and metadata are store
 ![Emulator](./screenshots/Emulator.png)
 
 ## Requirements
-* MySQL Server 8+
+* MySQL Server 8+*
 * Internet Game Database API Key. See: https://api-docs.igdb.com/#account-creation
 
+***Note**: MariaDB is currently not supported as Gaseous uses features present only in MySQL. This is being tracked in https://github.com/gaseous-project/gaseous-server/issues/93
+
 ## Third Party Projects
 The following projects are used by Gaseous
 * https://dotnet.microsoft.com/en-us/apps/aspnet
@@ -22,6 +32,8 @@ The following projects are used by Gaseous
 ## Discord Server
 [![Join our Discord server!](https://invite.casperiv.dev/?inviteCode=Nhu7wpT3k4&format=svg)](https://discord.gg/Nhu7wpT3k4)
 
+# Setup
+
 ## Configuration File
 When Gaseous-Server is started for the first time, it creates a configuration file at ~/.gaseous-server/config.json if it doesn't exist. Some values can be filled in using environment variables (such as in the case of using docker).
 
@@ -60,45 +72,110 @@ When Gaseous-Server is started for the first time, it creates a configuration fi
 
 ```
 
-## Deploy with Docker
+## Docker
+### Deploy with the prebuilt Docker image
 Dockerfile and docker-compose.yml files have been provided to make deployment of the server as easy as possible.
 1. Download the docker-compose.yml file
 2. Open the docker-compose.yml file and edit the igdbclientid and igdbclientsecret to the values retrieved from your IGDB account
-3. Run the command "docker-compose up -d"
+3. Run the command ```docker-compose up -d```
 4. Connect to the host on port 5198
 
-## Build and Deploy with Docker
+### Build and deploy a Docker image from source
 Dockerfile and docker-compose-build.yml files have been provided to make deployment of the server as easy as possible.
-1. Clone the repo with "git clone https://github.com/gaseous-project/gaseous-server.git"
+1. Clone the repo with ```git clone https://github.com/gaseous-project/gaseous-server.git```
 2. Change into the gaseous-server directory
-3. Open the docker-compose-build.yml file and edit the igdbclientid and igdbclientsecret to the values retrieved from your IGDB account
-4. Run the command "docker-compose up --file docker-compose-build.yml -d"
-5. Connect to the host on port 5198
+3. Clone the submodules with the command ```git submodule update --init```
+4. Open the docker-compose-build.yml file and edit the igdbclientid and igdbclientsecret to the values retrieved from your IGDB account
+5. Run the command ```docker-compose --file docker-compose-build.yml up -d```
+6. Connect to the host on port 5198
 
-## Adding Content
-While games can be added to the server without them, it is recommended adding some signature DAT files beforehand to allow for better matching of ROM to game.
+## Source
+### Build and deploy
+1. Install and configure a MySQL instance
+2. Install the dotnet 7.0 packages appropriate for your operating system
+    * See: https://learn.microsoft.com/en-us/dotnet/core/install/linux
+3. Create a database user with permission to create a databse. Gaseous will create the new database and apply the database schema on it's first startup.
+4. Clone the repo with ```git clone https://github.com/gaseous-project/gaseous-server.git```
+5. Change into the gaseous-server directory
+6. As the main branch is the development branch, you might want to change to a stable version - these are tagged with a version number. For example to change to the 1.5.0 release, use the command ```git checkout v1.5.0```
+    * Check the releases page for the version you would like to run: https://github.com/gaseous-project/gaseous-server/releases
+7. Clone the submodules with the command ```git submodule update --init --recursive```
+    * This command will clone the code that the server uses from other projects (currently only EmulatorJS)
+8. Create a directory in the home directory of the user that will run the server. For example, if running as the user ```gaseous```, create the directory ```/home/gaseous/.gaseous-server```
+9. Change into the ```.gaseous-server``` directory created in the previous step
+10. Copy the JSON from the config file above into a new file named ```config.json```
+11. Update the database section with the database server hostname, username, password, and port
+12. Compile the server by changing back to the repo cloned earlier and executing:
+    * ```dotnet restore "gaseous-server/gaseous-server.csproj"```
+    * ```dotnet publish "gaseous-server/gaseous-server.csproj" --use-current-runtime --self-contained false -c Release -o <output directory>```
+    * replace ```<output directory>``` with the directory of your choosing. The compiled application will be copied there. For this example we'll use ```/opt/gaseous-server```
+13. The server can then be started by executing ```dotnet /opt/gaseous-server/gaseous-server.dll```
+    * If you would like the server to run on a different ip address and port (for example 0.0.0.0:8080), add the --urls argument: ```dotnet /opt/gaseous-server/gaseous-server.dll --urls http://0.0.0.0:8080```
+
+**Note**: The above instructions were tested on macOS Ventura, and Ubuntu 22.04.3. There was a report that Debian 11 had an issue with the git submodule commands (see: https://github.com/gaseous-project/gaseous-server/issues/71). This was possibly due to an older git package.
+
+If the git submodule commands aren't working, you can:
+1. change to the ```gaseous-server/wwwroot/emulators``` directory
+2. delete the ```EmulatorJS``` directory
+3. clone the EmulatorJS repository with ```git clone https://github.com/EmulatorJS/EmulatorJS.git```
+
+### Updating from source
+1. Stop the server
+2. Switch to the source directory
+3. Update your repo:
+    * If running from the main branch, run ```git pull``` to update the repo
+    * If running from another branch or tag, run:
+      * ```git fetch```
+      * ```git checkout <branch or tag name>```
+4. Update the submodules with ```git submodule update --recursive```
+5. Run steps 12 and 13 from the above Build guide
+
+# Adding Content
+While games can be added to the server without them, it is recommended adding some signature DAT files beforehand to allow for better matching of ROMs to games.
 
 These signature DAT files contain a list of titles with hashes for many of the ROM images that have been found by the community.
 
-Currently only TOSEC is supported, though more will be added.
+Currently supported DAT's:
+* TOSEC: https://www.tosecdev.org/downloads/category/56-2023-01-23
+* MAME Arcade and MAME Mess: https://www.progettosnaps.net/dats/MAME
 
-### Adding signature DAT files
+If there are other DAT's you'd like to see support for, please raise an issue with a link to the DAT's.
+
+## Adding signature DAT files
+### TOSEC
 1. Download the DAT files from the source website. For example; from https://www.tosecdev.org/downloads/category/56-2023-01-23
 2. Extract the archive
 3. Copy the DAT files to ~/.gaseous-server/Data/Signatures/TOSEC/
 
-### Adding game image files
-1. Ensure your game image file is unzipped, and clearly named. Attempting a search for the game name on https://www.igdb.com can help with file naming. If a hash search is unsuccessful, Gaseous will fall back to attempting to search by the file name.
-2. Copy the file to ~/.gaseous-server/Data/Import
+### MAME Arcade
+1. Download the DAT files from the source website. For example; from https://www.progettosnaps.net/dats/MAME
+2. Extract the archive
+3. Copy the file name "MAME 0.257 (arcade).dat" files to ~/.gaseous-server/Data/Signatures/MAME Arcade/
 
+### MAME MESS
+1. Download the DAT files from the source website. For example; from https://www.progettosnaps.net/dats/MAME
+2. Extract the archive
+3. Copy the file name "MAME 0.257 (mess).dat" files to ~/.gaseous-server/Data/Signatures/MAME MESS/
+
+# Adding Game Images
+1. Files can be presented as either stand alone files, or as zip files - currently 7z is unsupported.
+2. Name the file appropriately.
+   * Attempting a search for the game name on https://www.igdb.com can help with file naming. If a hash search is unsuccessful, Gaseous will fall back to attempting to search by the file name.
+3. Add the file to the server:
+   * Click the Upload button in the top right of the main Gaseous web page, and drag the files into the modal. The files will be uploaded and analyzed.
+   * Copy the file to ~/.gaseous-server/Data/Import
+
+# Game Image Title Matching
 Image to game matching follows the following order of operations, stopping the process at the first match:
 ### Get the file signature
 1. Attempt a hash search
 2. Attempt to search the signature database for a rom matching the file name - sometimes the hash can not be matched as a highscore table for example was saved to the image
 3. Attempt to parse the file name - clues such as the extension being used to define which platform the file belongs to are used to create a search criteria
 
+**Note**: If the file being scanned is a zip, the file will be extracted and searched. The first file whose signature can be found will be used to match the entire zip archive - be sure that the zip only contains files related to one game.
+
 ### Create a list of search candidates
-Before beginning, remove any version numbers.
+Before beginning, remove any version numbers, and anything in the search string that is between ()
 1. Add the full name of the image
 2. Add the name of the image with any " - " replaced by ": "
 3. Add the name of the image with text after a " - " removed
@@ -109,5 +186,8 @@ Loop through each of the search candidates searching using:
 1. "where" - exact match as the search candidate
 2. "wherefuzzy" - partial match using wildcards
 3. "search" - uses a more flexible search method
+4. "searchNoPlatform" - uses the "search" method, but does not constrain the search to the determined platform
 
-Note: that if more than one result is found, the image will be set as "Unknown" as there is no way for Gaseous to know which title is the correct one.
+**Note**: If more than one result is found, the seach will loop through the returned results:
+* If an exact (case-insensitive) match is found, that result is used for the match
+* If still no match, the image will be set as "Unknown" as there is no way for Gaseous to know which title is the correct one.

gaseous-identifier/Program.cs

@@ -1,183 +0,0 @@
-// parse command line
-using System.Security.Cryptography;
-using System.Text;
-using System.Xml;
-using System.Xml.Serialization;
-using Newtonsoft.Json;
-using gaseous_romsignatureobject;
-using gaseous_signature_parser.parsers;
-
-string[] commandLineArgs = Environment.GetCommandLineArgs();
-
-string scanPath = "./";
-string tosecXML = "";
-
-string inArgument = "";
-foreach (string commandLineArg in commandLineArgs)
-{
-    if (commandLineArg != commandLineArgs[0])
-    {
-        if (inArgument == "")
-        {
-            switch (commandLineArg.ToLower())
-            {
-                case "-scanpath":
-                    inArgument = commandLineArg.ToLower();
-                    break;
-                case "-tosecpath":
-                    inArgument = commandLineArg.ToLower();
-                    break;
-                default:
-                    break;
-            }
-        }
-        else
-        {
-            switch (inArgument)
-            {
-                case "-scanpath":
-                    scanPath = commandLineArg;
-                    break;
-                case "-tosecpath":
-                    tosecXML = commandLineArg;
-                    break;
-                default:
-                    break;
-            }
-            inArgument = "";
-        }
-    }
-}
-
-scanPath = Path.GetFullPath(scanPath);
-Console.WriteLine("ROM search path: " + scanPath);
-
-List<RomSignatureObject> romSignatures = new List<RomSignatureObject>();
-System.Collections.ArrayList availablePlatforms = new System.Collections.ArrayList();
-
-// load TOSEC XML files
-if (tosecXML != null && tosecXML.Length > 0)
-{
-    tosecXML = Path.GetFullPath(tosecXML);
-    Console.WriteLine("TOSEC is enabled");
-    Console.WriteLine("TOSEC XML search path: " + tosecXML);
-
-    string[] tosecPathContents = Directory.GetFiles(tosecXML);
-    int lastCLILineLength = 0;
-    for (UInt16 i = 0; i < tosecPathContents.Length; ++i)
-    {
-        string tosecXMLFile = tosecPathContents[i];
-
-        TosecParser tosecParser = new TosecParser();
-        RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile);
-
-        string statusOutput = i + " / " + tosecPathContents.Length + " : " + Path.GetFileName(tosecXMLFile);
-        Console.Write("\r " + statusOutput.PadRight(lastCLILineLength, ' ') + "\r");
-        lastCLILineLength = statusOutput.Length;
-
-        foreach (RomSignatureObject.Game gameRom in tosecObject.Games)
-        {
-            if (!availablePlatforms.Contains(gameRom.System))
-            {
-                availablePlatforms.Add(gameRom.System);
-            }
-        }
-
-        romSignatures.Add(tosecObject);
-    }
-    Console.WriteLine("");
-} else
-{
-    Console.WriteLine("TOSEC is disabled.");
-}
-Console.WriteLine(romSignatures.Count + " TOSEC files loaded");
-
-// Summarise signatures
-if (availablePlatforms.Count > 0)
-{
-    availablePlatforms.Sort();
-    Console.WriteLine("Platforms loaded:");
-    foreach (string platform in availablePlatforms)
-    {
-        Console.WriteLine(" * " + platform);
-    }
-}
-
-Console.WriteLine("Examining files");
-string[] romPathContents = Directory.GetFiles(scanPath);
-foreach (string romFile in romPathContents)
-{
-    Console.WriteLine("Checking " + romFile);
-
-    var stream = File.OpenRead(romFile);
-
-    var md5 = MD5.Create();
-    byte[] md5HashByte = md5.ComputeHash(stream);
-    string md5Hash = BitConverter.ToString(md5HashByte).Replace("-", "").ToLowerInvariant();
-
-    var sha1 = SHA1.Create();
-    byte[] sha1HashByte = sha1.ComputeHash(stream);
-    string sha1Hash = BitConverter.ToString(sha1HashByte).Replace("-", "").ToLowerInvariant();
-
-    bool gameFound = false;
-    foreach (RomSignatureObject tosecList in romSignatures)
-    {
-        foreach (RomSignatureObject.Game gameObject in tosecList.Games)
-        {
-            foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
-            {
-                if (romObject.Md5 != null)
-                {
-                    if (md5Hash == romObject.Md5.ToLowerInvariant())
-                    {
-                        // match
-                        gameFound = true;
-                    }
-                }
-                if (romObject.Sha1 != null)
-                {
-                    if (md5Hash == romObject.Sha1.ToLowerInvariant())
-                    {
-                        // match
-                        gameFound = true;
-                    }
-                }
-                if (gameFound == true)
-                {
-                    Console.WriteLine(romObject.Name);
-
-                    RomSignatureObject.Game gameSignature = gameObject;
-                    gameSignature.Roms.Clear();
-                    gameSignature.Roms.Add(romObject);
-
-                    var jsonSerializerSettings = new JsonSerializerSettings();
-                    jsonSerializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
-                    jsonSerializerSettings.NullValueHandling = NullValueHandling.Ignore;
-                    Console.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(gameSignature, Newtonsoft.Json.Formatting.Indented, jsonSerializerSettings));
-                    break;
-                }
-            }
-            if (gameFound == true) { break; }
-        }
-        if (gameFound == true) { break; }
-    }
-    if (gameFound == false)
-    {
-        Console.WriteLine("File not found in TOSEC library");
-    }
-}
-
-string SearchTitle = "California Games";
-foreach (RomSignatureObject romSignatureObject in romSignatures)
-{
-    foreach (RomSignatureObject.Game gameObject in romSignatureObject.Games)
-    {
-        if (gameObject.Name == SearchTitle)
-        {
-            var jsonSerializerSettings = new JsonSerializerSettings();
-            jsonSerializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
-            jsonSerializerSettings.NullValueHandling = NullValueHandling.Ignore;
-            Console.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(gameObject, Newtonsoft.Json.Formatting.Indented, jsonSerializerSettings));
-        }
-    }
-}

gaseous-identifier/gaseous-identifier-testapp.csproj

@@ -1,21 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
-    <RootNamespace>gaseous_identifier</RootNamespace>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <None Remove="Newtonsoft.Json" />
-  </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\gaseous-romsignatureobject\gaseous-romsignatureobject.csproj" />
-    <ProjectReference Include="..\gaseous-signature-parser\gaseous-signature-parser.csproj" />
-  </ItemGroup>
-</Project>

gaseous-romsignatureobject/RomSignatureObject.cs

@@ -1,125 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace gaseous_romsignatureobject
-{
-    /// <summary>
-    /// Object returned by all signature engines containing metadata about the ROM's in the data files
-    ///
-    /// This class was based on the TOSEC dataset, so may need to be expanded as new signature engines are added
-    /// </summary>
-	public class RomSignatureObject
-    {
-        public string? Name { get; set; }
-        public string? Description { get; set; }
-        public string? Category { get; set; }
-        public string? Version { get; set; }
-        public string? Author { get; set; }
-        public string? Email { get; set; }
-        public string? Homepage { get; set; }
-        public Uri? Url { get; set; }
-        public string? SourceType { get; set; }
-        public string SourceMd5 { get; set; } = "";
-        public string SourceSHA1 { get; set; } = "";
-
-        public List<Game> Games { get; set; } = new List<Game>();
-
-        public class Game
-        {
-            public string? Name { get; set; }
-            public string? Description { get; set; }
-            public string? Year { get; set; }
-            public string? Publisher { get; set; }
-            public DemoTypes Demo { get; set; }
-            public string? System { get; set; }
-            public string? SystemVariant { get; set; }
-            public string? Video { get; set; }
-            public string? Country { get; set; }
-            public string? Language { get; set; }
-            public string? Copyright { get; set; }
-            public List<Rom> Roms { get; set; } = new List<Rom>();
-            public int RomCount
-            {
-                get
-                {
-                    return Roms.Count();
-                }
-            }
-
-            public enum DemoTypes
-            {
-                NotDemo = 0,
-                demo = 1,
-                demo_kiosk = 2,
-                demo_playable = 3,
-                demo_rolling = 4,
-                demo_slideshow = 5
-            }
-
-            public class Rom
-            {
-                public string? Name { get; set; }
-                public UInt64? Size { get; set; }
-                public string? Crc { get; set; }
-                public string? Md5 { get; set; }
-                public string? Sha1 { get; set; }
-
-                public string? DevelopmentStatus { get; set; }
-
-                public List<string> flags { get; set; } = new List<string>();
-
-                public RomTypes RomType { get; set; }
-                public string? RomTypeMedia { get; set; }
-                public string? MediaLabel { get; set; }
-
-                public SignatureSourceType SignatureSource { get; set; }
-
-                public enum SignatureSourceType
-                {
-                    None = 0,
-                    TOSEC = 1
-                }
-
-                public enum RomTypes
-                {
-                    /// <summary>
-                    /// Media type is unknown
-                    /// </summary>
-                    Unknown = 0,
-
-                    /// <summary>
-                    /// Optical media
-                    /// </summary>
-                    Disc = 1,
-
-                    /// <summary>
-                    /// Magnetic media
-                    /// </summary>
-                    Disk = 2,
-
-                    /// <summary>
-                    /// Individual files
-                    /// </summary>
-                    File = 3,
-
-                    /// <summary>
-                    /// Individual pars
-                    /// </summary>
-                    Part = 4,
-
-                    /// <summary>
-                    /// Tape base media
-                    /// </summary>
-                    Tape = 5,
-
-                    /// <summary>
-                    /// Side of the media
-                    /// </summary>
-                    Side = 6
-                }
-            }
-
-        }
-    }
-}
-

gaseous-romsignatureobject/gaseous-romsignatureobject.csproj

@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <RootNamespace>gaseous_romsignatureobject</RootNamespace>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
-    <WarningLevel>4</WarningLevel>
-  </PropertyGroup>
-</Project>

gaseous-server/Classes/Bios.cs

@@ -16,16 +16,13 @@ namespace gaseous_server.Classes
         {
             foreach (Models.PlatformMapping.PlatformMapItem platformMapping in Models.PlatformMapping.PlatformMap)
             {
-                if (platformMapping.WebEmulator != null)
+                if (platformMapping.Bios != null)
                 {
-                    if (platformMapping.WebEmulator.Bios != null)
+                    foreach (Models.PlatformMapping.PlatformMapItem.EmulatorBiosItem emulatorBiosItem in platformMapping.Bios)
                     {
-                        foreach (Models.PlatformMapping.PlatformMapItem.WebEmulatorItem.EmulatorBiosItem emulatorBiosItem in platformMapping.WebEmulator.Bios)
+                        if (emulatorBiosItem.hash.ToLower() == MD5.ToLower())
                         {
-                            if (emulatorBiosItem.hash.ToLower() == MD5.ToLower())
-                            {
-                                return platformMapping;
-                            }
+                            return platformMapping;
                         }
                     }
                 }
@@ -69,33 +66,29 @@ namespace gaseous_server.Classes
 
             foreach (Models.PlatformMapping.PlatformMapItem platformMapping in Models.PlatformMapping.PlatformMap)
             {
-                if (platformMapping.WebEmulator != null)
-                    {
-                    if (platformMapping.WebEmulator.Bios != null)
-                    {
-                        IGDB.Models.Platform platform = Metadata.Platforms.GetPlatform(platformMapping.IGDBId);
+                if (platformMapping.Bios != null)
+                {
+                    IGDB.Models.Platform platform = Metadata.Platforms.GetPlatform(platformMapping.IGDBId);
 
-                        foreach (Models.PlatformMapping.PlatformMapItem.WebEmulatorItem.EmulatorBiosItem emulatorBios in platformMapping.WebEmulator.Bios)
+                    foreach (Models.PlatformMapping.PlatformMapItem.EmulatorBiosItem emulatorBios in platformMapping.Bios)
+                    {
+                        BiosItem biosItem = new BiosItem
                         {
-                            BiosItem biosItem = new BiosItem
-                            {
-                                platformid = platformMapping.IGDBId,
-                                platformslug = platform.Slug,
-                                platformname = platform.Name,
-                                description = emulatorBios.description,
-                                filename = emulatorBios.filename,
-                                region = emulatorBios.region,
-                                hash = emulatorBios.hash
-                            };
-                            biosItems.Add(biosItem);
-                        }
+                            platformid = platformMapping.IGDBId,
+                            platformslug = platform.Slug,
+                            platformname = platform.Name,
+                            description = emulatorBios.description,
+                            filename = emulatorBios.filename,
+                            hash = emulatorBios.hash.ToLower()
+                        };
+                        biosItems.Add(biosItem);
                     }
                 }
             }
             return biosItems;
         }
 
-        public class BiosItem : Models.PlatformMapping.PlatformMapItem.WebEmulatorItem.EmulatorBiosItem
+        public class BiosItem : Models.PlatformMapping.PlatformMapItem.EmulatorBiosItem
         {
             public long platformid { get; set; }
             public string platformslug { get; set; }

gaseous-server/Classes/Collections.cs

@@ -6,6 +6,7 @@ using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using gaseous_server.Classes.Metadata;
 using gaseous_server.Controllers;
+using gaseous_server.Models;
 using gaseous_tools;
 using IGDB.Models;
 using Newtonsoft.Json;
@@ -57,7 +58,7 @@ namespace gaseous_server.Classes
         public static CollectionItem NewCollection(CollectionItem item)
         {
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-            string sql = "INSERT INTO RomCollections (`Name`, Description, Platforms, Genres, Players, PlayerPerspectives, Themes, MinimumRating, MaximumRating, MaximumRomsPerPlatform, MaximumBytesPerPlatform, MaximumCollectionSizeInBytes, BuiltStatus) VALUES (@name, @description, @platforms, @genres, @players, @playerperspectives, @themes, @minimumrating, @maximumrating, @maximumromsperplatform, @maximumbytesperplatform, @maximumcollectionsizeinbytes, @builtstatus); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
+            string sql = "INSERT INTO RomCollections (`Name`, Description, Platforms, Genres, Players, PlayerPerspectives, Themes, MinimumRating, MaximumRating, MaximumRomsPerPlatform, MaximumBytesPerPlatform, MaximumCollectionSizeInBytes, FolderStructure, IncludeBIOSFiles, AlwaysInclude, BuiltStatus) VALUES (@name, @description, @platforms, @genres, @players, @playerperspectives, @themes, @minimumrating, @maximumrating, @maximumromsperplatform, @maximumbytesperplatform, @maximumcollectionsizeinbytes, @folderstructure, @includebiosfiles, @alwaysinclude, @builtstatus); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
             Dictionary<string, object> dbDict = new Dictionary<string, object>();
             dbDict.Add("name", item.Name);
             dbDict.Add("description", item.Description);
@@ -71,6 +72,9 @@ namespace gaseous_server.Classes
             dbDict.Add("maximumromsperplatform", Common.ReturnValueIfNull(item.MaximumRomsPerPlatform, -1));
             dbDict.Add("maximumbytesperplatform", Common.ReturnValueIfNull(item.MaximumBytesPerPlatform, -1));
             dbDict.Add("maximumcollectionsizeinbytes", Common.ReturnValueIfNull(item.MaximumCollectionSizeInBytes, -1));
+            dbDict.Add("folderstructure", Common.ReturnValueIfNull(item.FolderStructure, CollectionItem.FolderStructures.Gaseous));
+            dbDict.Add("includebiosfiles", Common.ReturnValueIfNull(item.IncludeBIOSFiles, 0));
+            dbDict.Add("alwaysinclude", Newtonsoft.Json.JsonConvert.SerializeObject(Common.ReturnValueIfNull(item.AlwaysInclude, new List<CollectionItem.AlwaysIncludeItem>())));
             dbDict.Add("builtstatus", CollectionItem.CollectionBuildStatus.WaitingForBuild);
             DataTable romDT = db.ExecuteCMD(sql, dbDict);
             long CollectionId = (long)romDT.Rows[0][0];
@@ -82,10 +86,10 @@ namespace gaseous_server.Classes
             return collectionItem;
         }
 
-        public static CollectionItem EditCollection(long Id, CollectionItem item)
+        public static CollectionItem EditCollection(long Id, CollectionItem item, bool ForceRebuild = true)
         {
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-            string sql = "UPDATE RomCollections SET `Name`=@name, Description=@description, Platforms=@platforms, Genres=@genres, Players=@players, PlayerPerspectives=@playerperspectives, Themes=@themes, MinimumRating=@minimumrating, MaximumRating=@maximumrating, MaximumRomsPerPlatform=@maximumromsperplatform, MaximumBytesPerPlatform=@maximumbytesperplatform, MaximumCollectionSizeInBytes=@maximumcollectionsizeinbytes, BuiltStatus=@builtstatus WHERE Id=@id";
+            string sql = "UPDATE RomCollections SET `Name`=@name, Description=@description, Platforms=@platforms, Genres=@genres, Players=@players, PlayerPerspectives=@playerperspectives, Themes=@themes, MinimumRating=@minimumrating, MaximumRating=@maximumrating, MaximumRomsPerPlatform=@maximumromsperplatform, MaximumBytesPerPlatform=@maximumbytesperplatform, MaximumCollectionSizeInBytes=@maximumcollectionsizeinbytes, FolderStructure=@folderstructure, IncludeBIOSFiles=@includebiosfiles, AlwaysInclude=@alwaysinclude, BuiltStatus=@builtstatus WHERE Id=@id";
             Dictionary<string, object> dbDict = new Dictionary<string, object>();
             dbDict.Add("id", Id);
             dbDict.Add("name", item.Name);
@@ -100,18 +104,39 @@ namespace gaseous_server.Classes
             dbDict.Add("maximumromsperplatform", Common.ReturnValueIfNull(item.MaximumRomsPerPlatform, -1));
             dbDict.Add("maximumbytesperplatform", Common.ReturnValueIfNull(item.MaximumBytesPerPlatform, -1));
             dbDict.Add("maximumcollectionsizeinbytes", Common.ReturnValueIfNull(item.MaximumCollectionSizeInBytes, -1));
-            dbDict.Add("builtstatus", CollectionItem.CollectionBuildStatus.WaitingForBuild);
-            db.ExecuteCMD(sql, dbDict);
-
+            dbDict.Add("folderstructure", Common.ReturnValueIfNull(item.FolderStructure, CollectionItem.FolderStructures.Gaseous));
+            dbDict.Add("includebiosfiles", Common.ReturnValueIfNull(item.IncludeBIOSFiles, 0));
+            dbDict.Add("alwaysinclude", Newtonsoft.Json.JsonConvert.SerializeObject(Common.ReturnValueIfNull(item.AlwaysInclude, new List<CollectionItem.AlwaysIncludeItem>())));
+            
             string CollectionZipFile = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, Id + ".zip");
-            if (File.Exists(CollectionZipFile))
+            if (ForceRebuild == true) 
             {
-                File.Delete(CollectionZipFile);
+                dbDict.Add("builtstatus", CollectionItem.CollectionBuildStatus.WaitingForBuild);
+                if (File.Exists(CollectionZipFile))
+                {
+                    Logging.Log(Logging.LogType.Warning, "Collections", "Deleting existing build of collection: " + item.Name);
+                    File.Delete(CollectionZipFile);
+                }
             }
-
+            else
+            {
+                if (File.Exists(CollectionZipFile))
+                {
+                    dbDict.Add("builtstatus", CollectionItem.CollectionBuildStatus.Completed);
+                }
+                else
+                {
+                    dbDict.Add("builtstatus", CollectionItem.CollectionBuildStatus.NoStatus);
+                }
+            }
+            db.ExecuteCMD(sql, dbDict);
+            
             CollectionItem collectionItem = GetCollection(Id);
 
-            StartCollectionItemBuild(Id);
+            if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild)
+            {
+                StartCollectionItemBuild(Id);
+            }
 
             return collectionItem;
         }
@@ -146,13 +171,10 @@ namespace gaseous_server.Classes
                 db.ExecuteCMD(sql, dbDict);
 
                 // start background task
-                foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
-                {
-                    if (qi.ItemType == ProcessQueue.QueueItemType.CollectionCompiler) { 
-                        qi.ForceExecute();
-                        break;
-                    }
-                }
+                ProcessQueue.QueueItem queueItem = new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.CollectionCompiler, 1, false, true);
+                queueItem.Options = Id;
+                queueItem.ForceExecute();
+                ProcessQueue.QueueItems.Add(queueItem);
             }
         }
 
@@ -160,9 +182,32 @@ namespace gaseous_server.Classes
             List<CollectionContents.CollectionPlatformItem> collectionPlatformItems = new List<CollectionContents.CollectionPlatformItem>();
 
             // get platforms
+            List<long> platformids = new List<long>();
+            platformids.AddRange(collectionItem.Platforms);
+
+            List<long>? DynamicPlatforms = new List<long>();
+            DynamicPlatforms.AddRange(collectionItem.Platforms);
+
             List<Platform> platforms = new List<Platform>();
-            if (collectionItem.Platforms.Count > 0) {
-                foreach (long PlatformId in collectionItem.Platforms) {
+
+            // add platforms with an inclusion status
+            foreach (CollectionItem.AlwaysIncludeItem alwaysIncludeItem in collectionItem.AlwaysInclude) 
+            {
+                if (
+                        alwaysIncludeItem.InclusionState == CollectionItem.AlwaysIncludeStatus.AlwaysInclude ||
+                        alwaysIncludeItem.InclusionState == CollectionItem.AlwaysIncludeStatus.AlwaysExclude
+                    )
+                    {
+                        if (!platformids.Contains(alwaysIncludeItem.PlatformId))
+                        {
+                            platformids.Add(alwaysIncludeItem.PlatformId);
+                        }
+                }
+            }
+
+            // add dynamic platforms
+            if (DynamicPlatforms.Count > 0) {
+                foreach (long PlatformId in platformids) {
                     platforms.Add(Platforms.GetPlatform(PlatformId));
                 }
             } else {
@@ -178,64 +223,114 @@ namespace gaseous_server.Classes
                 long TotalRomSize = 0;
                 long TotalGameCount = 0;
 
-                List<Game> games = GamesController.GetGames("",
-                    platform.Id.ToString(),
-                    string.Join(",", collectionItem.Genres),
-                    string.Join(",", collectionItem.Players),
-                    string.Join(",", collectionItem.PlayerPerspectives),
-                    string.Join(",", collectionItem.Themes),
-                    collectionItem.MinimumRating,
-                    collectionItem.MaximumRating
-                );
+                bool isDynamic = false;
+                if (DynamicPlatforms.Contains((long)platform.Id))
+                {
+                    isDynamic = true;
+                }
+                else if (DynamicPlatforms.Count == 0)
+                {
+                    isDynamic = true;
+                }
+
+                List<Game> games = new List<Game>();
+                if (isDynamic == true)
+                {
+                    games = GamesController.GetGames("",
+                        platform.Id.ToString(),
+                        string.Join(",", collectionItem.Genres),
+                        string.Join(",", collectionItem.Players),
+                        string.Join(",", collectionItem.PlayerPerspectives),
+                        string.Join(",", collectionItem.Themes),
+                        collectionItem.MinimumRating,
+                        collectionItem.MaximumRating
+                    );
+                }
 
                 CollectionContents.CollectionPlatformItem collectionPlatformItem = new CollectionContents.CollectionPlatformItem(platform);
                 collectionPlatformItem.Games = new List<CollectionContents.CollectionPlatformItem.CollectionGameItem>();
 
-                foreach (Game game in games) {
-                    CollectionContents.CollectionPlatformItem.CollectionGameItem collectionGameItem = new CollectionContents.CollectionPlatformItem.CollectionGameItem(game);
+                // add titles with an inclusion status
+                foreach (CollectionItem.AlwaysIncludeItem alwaysIncludeItem in collectionItem.AlwaysInclude) 
+                {
+                    if (
+                        (
+                            alwaysIncludeItem.InclusionState == CollectionItem.AlwaysIncludeStatus.AlwaysInclude ||
+                            alwaysIncludeItem.InclusionState == CollectionItem.AlwaysIncludeStatus.AlwaysExclude
+                        ) && alwaysIncludeItem.PlatformId == platform.Id
+                        ) 
+                        {
+                            Game AlwaysIncludeGame = Games.GetGame(alwaysIncludeItem.GameId, false, false, false);
+                            CollectionContents.CollectionPlatformItem.CollectionGameItem gameItem = new CollectionContents.CollectionPlatformItem.CollectionGameItem(AlwaysIncludeGame);
+                            gameItem.InclusionStatus = new CollectionItem.AlwaysIncludeItem();
+                            gameItem.InclusionStatus.PlatformId = alwaysIncludeItem.PlatformId;
+                            gameItem.InclusionStatus.GameId = alwaysIncludeItem.GameId;
+                            gameItem.InclusionStatus.InclusionState = alwaysIncludeItem.InclusionState;
+                            gameItem.Roms = Roms.GetRoms((long)gameItem.Id, (long)platform.Id);
 
-                    List<Roms.GameRomItem> gameRoms = Roms.GetRoms((long)game.Id, (long)platform.Id);
-                    
-                    bool AddGame = false;
-
-                    // calculate total rom size for the game
-                    long GameRomSize = 0;
-                    foreach (Roms.GameRomItem gameRom in gameRoms) {
-                        GameRomSize += gameRom.Size;
+                            collectionPlatformItem.Games.Add(gameItem);
                     }
-                    if (collectionItem.MaximumBytesPerPlatform > 0) {
-                        if ((TotalRomSize + GameRomSize) < collectionItem.MaximumBytesPerPlatform) {
-                            AddGame = true;
+                }
+
+                foreach (Game game in games) {
+                    bool gameAlreadyInList = false;
+                    foreach (CollectionContents.CollectionPlatformItem.CollectionGameItem existingGame in collectionPlatformItem.Games) 
+                    {
+                        if (existingGame.Id == game.Id) 
+                        {
+                            gameAlreadyInList = true;
                         }
                     }
-                    else 
+
+                    if (gameAlreadyInList == false)
                     {
-                        AddGame = true;
-                    }
+                        CollectionContents.CollectionPlatformItem.CollectionGameItem collectionGameItem = new CollectionContents.CollectionPlatformItem.CollectionGameItem(game);
 
-                    if (AddGame == true) {
-                        TotalRomSize += GameRomSize;
+                        List<Roms.GameRomItem> gameRoms = Roms.GetRoms((long)game.Id, (long)platform.Id);
+                        
+                        bool AddGame = false;
 
-                        bool AddRoms = false;
-
-                        if (collectionItem.MaximumRomsPerPlatform > 0) { 
-                            if (TotalGameCount < collectionItem.MaximumRomsPerPlatform) {
-                                AddRoms = true;
+                        // calculate total rom size for the game
+                        long GameRomSize = 0;
+                        foreach (Roms.GameRomItem gameRom in gameRoms) {
+                            GameRomSize += gameRom.Size;
+                        }
+                        if (collectionItem.MaximumBytesPerPlatform > 0) {
+                            if ((TotalRomSize + GameRomSize) < collectionItem.MaximumBytesPerPlatform) {
+                                AddGame = true;
                             }
                         }
-                        else
+                        else 
                         {
-                            AddRoms = true;
+                            AddGame = true;
                         }
 
-                        if (AddRoms == true) {
-                            TotalGameCount += 1;
-                            collectionGameItem.Roms = gameRoms;
-                            collectionPlatformItem.Games.Add(collectionGameItem);
+                        if (AddGame == true) {
+                            TotalRomSize += GameRomSize;
+
+                            bool AddRoms = false;
+
+                            if (collectionItem.MaximumRomsPerPlatform > 0) { 
+                                if (TotalGameCount < collectionItem.MaximumRomsPerPlatform) {
+                                    AddRoms = true;
+                                }
+                            }
+                            else
+                            {
+                                AddRoms = true;
+                            }
+
+                            if (AddRoms == true) {
+                                TotalGameCount += 1;
+                                collectionGameItem.Roms = gameRoms;
+                                collectionPlatformItem.Games.Add(collectionGameItem);
+                            }
                         }
                     }
                 }
 
+                collectionPlatformItem.Games.Sort((x, y) => x.Name.CompareTo(y.Name));
+
                 if (collectionPlatformItem.Games.Count > 0)
                 {
                     bool AddPlatform = false;
@@ -258,109 +353,181 @@ namespace gaseous_server.Classes
                 }
             }
 
+            collectionPlatformItems.Sort((x, y) => x.Name.CompareTo(y.Name));
+
             CollectionContents collectionContents = new CollectionContents();
             collectionContents.Collection = collectionPlatformItems;
+
             return collectionContents;
         }
 
-        public static void CompileCollections()
+        public static void CompileCollections(long CollectionId)
         {
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 
-            List<CollectionItem> collectionItems = GetCollections();
-            foreach (CollectionItem collectionItem in collectionItems)
+            CollectionItem collectionItem = GetCollection(CollectionId);
+            if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild)
             {
-                if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild)
+                Logging.Log(Logging.LogType.Information, "Collections", "Beginning build of collection: " + collectionItem.Name);
+
+                // set starting
+                string sql = "UPDATE RomCollections SET BuiltStatus=@bs WHERE Id=@id";
+                Dictionary<string, object> dbDict = new Dictionary<string, object>();
+                dbDict.Add("id", collectionItem.Id);
+                dbDict.Add("bs", CollectionItem.CollectionBuildStatus.Building);
+                db.ExecuteCMD(sql, dbDict);
+
+                List<CollectionContents.CollectionPlatformItem> collectionPlatformItems = GetCollectionContent(collectionItem).Collection;
+                string ZipFilePath = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, collectionItem.Id + ".zip");
+                string ZipFileTempPath = Path.Combine(Config.LibraryConfiguration.LibraryTempDirectory, collectionItem.Id.ToString());
+
+                try
                 {
-                    // set starting
-                    string sql = "UPDATE RomCollections SET BuiltStatus=@bs WHERE Id=@id";
-                    Dictionary<string, object> dbDict = new Dictionary<string, object>();
-                    dbDict.Add("id", collectionItem.Id);
-                    dbDict.Add("bs", CollectionItem.CollectionBuildStatus.Building);
-                    db.ExecuteCMD(sql, dbDict);
-
-                    List<CollectionContents.CollectionPlatformItem> collectionPlatformItems = GetCollectionContent(collectionItem).Collection;
-                    string ZipFilePath = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, collectionItem.Id + ".zip");
-                    string ZipFileTempPath = Path.Combine(Config.LibraryConfiguration.LibraryTempDirectory, collectionItem.Id.ToString());
-
-                    try
+                    
+                    // clean up if needed
+                    if (File.Exists(ZipFilePath))
                     {
-                        
-                        // clean up if needed
-                        if (File.Exists(ZipFilePath))
-                        {
-                            File.Delete(ZipFilePath);
-                        }
+                        Logging.Log(Logging.LogType.Warning, "Collections", "Deleting existing build of collection: " + collectionItem.Name);
+                        File.Delete(ZipFilePath);
+                    }
 
-                        if (Directory.Exists(ZipFileTempPath))
-                        {
-                            Directory.Delete(ZipFileTempPath, true);
-                        }
+                    if (Directory.Exists(ZipFileTempPath))
+                    {
+                        Directory.Delete(ZipFileTempPath, true);
+                    }
 
-                        // gather collection files
-                        Directory.CreateDirectory(ZipFileTempPath);
+                    // gather collection files
+                    Directory.CreateDirectory(ZipFileTempPath);
+                    string ZipBiosPath = Path.Combine(ZipFileTempPath, "BIOS");
 
-                        foreach (CollectionContents.CollectionPlatformItem collectionPlatformItem in collectionPlatformItems)
+                    // get the games
+                    foreach (CollectionContents.CollectionPlatformItem collectionPlatformItem in collectionPlatformItems)
+                    {
+                        // get platform bios files if present
+                        if (collectionItem.IncludeBIOSFiles == true)
                         {
-                            // create platform directory
-                            string ZipPlatformPath = Path.Combine(ZipFileTempPath, collectionPlatformItem.Slug);
-                            if (!Directory.Exists(ZipPlatformPath))
-                            {
-                                Directory.CreateDirectory(ZipPlatformPath);
+                            List<Bios.BiosItem> bios = Bios.GetBios(collectionPlatformItem.Id, true);
+                            if (!Directory.Exists(ZipBiosPath)) {
+                                Directory.CreateDirectory(ZipBiosPath);
                             }
 
-                            foreach (CollectionContents.CollectionPlatformItem.CollectionGameItem collectionGameItem in collectionPlatformItem.Games)
+                            foreach (Bios.BiosItem biosItem in bios) 
                             {
-                                // create game directory
-                                string ZipGamePath = Path.Combine(ZipPlatformPath, collectionGameItem.Slug);
-                                if (!Directory.Exists(ZipGamePath))
+                                if (File.Exists(biosItem.biosPath))
                                 {
-                                    Directory.CreateDirectory(ZipGamePath);
+                                    Logging.Log(Logging.LogType.Information, "Collections", "Copying BIOS file: " + biosItem.filename);
+                                    File.Copy(biosItem.biosPath, Path.Combine(ZipBiosPath, biosItem.filename));
+                                }
+                            }
+                        }
+
+                        // create platform directory
+                        string ZipPlatformPath = "";
+                        switch (collectionItem.FolderStructure)
+                        {
+                            case CollectionItem.FolderStructures.Gaseous:
+                                ZipPlatformPath = Path.Combine(ZipFileTempPath, collectionPlatformItem.Slug);
+                                break;
+
+                            case CollectionItem.FolderStructures.RetroPie:
+                                try
+                                {
+                                    PlatformMapping.PlatformMapItem platformMapItem = PlatformMapping.GetPlatformMap(collectionPlatformItem.Id);
+                                    ZipPlatformPath = Path.Combine(ZipFileTempPath, "roms", platformMapItem.RetroPieDirectoryName);
+                                }
+                                catch
+                                {
+                                    ZipPlatformPath = Path.Combine(ZipFileTempPath, collectionPlatformItem.Slug);
                                 }
 
+                                break;
+
+                        }
+                        if (!Directory.Exists(ZipPlatformPath))
+                        {
+                            Directory.CreateDirectory(ZipPlatformPath);
+                        }
+
+                        foreach (CollectionContents.CollectionPlatformItem.CollectionGameItem collectionGameItem in collectionPlatformItem.Games)
+                        {
+                            bool includeGame = false;
+                            if (collectionGameItem.InclusionStatus == null)
+                            {
+                                includeGame = true;
+                            }
+                            else
+                            {
+                                if (collectionGameItem.InclusionStatus.InclusionState == CollectionItem.AlwaysIncludeStatus.AlwaysInclude)
+                                {
+                                    includeGame = true;
+                                }
+                            }
+
+                            if (includeGame == true)
+                            {
+                                string ZipGamePath = "";
+                                switch (collectionItem.FolderStructure)
+                                {
+                                    case CollectionItem.FolderStructures.Gaseous:
+                                        // create game directory
+                                        ZipGamePath = Path.Combine(ZipPlatformPath, collectionGameItem.Slug);
+                                        if (!Directory.Exists(ZipGamePath))
+                                        {
+                                            Directory.CreateDirectory(ZipGamePath);
+                                        }
+                                        break;
+
+                                    case CollectionItem.FolderStructures.RetroPie:
+                                        ZipGamePath = ZipPlatformPath;
+                                        break;
+                                }                                    
+                                
                                 // copy in roms
                                 foreach (Roms.GameRomItem gameRomItem in collectionGameItem.Roms)
                                 {
                                     if (File.Exists(gameRomItem.Path))
                                     {
+                                        Logging.Log(Logging.LogType.Information, "Collections", "Copying ROM: " + gameRomItem.Name);
                                         File.Copy(gameRomItem.Path, Path.Combine(ZipGamePath, gameRomItem.Name));
                                     }
                                 }
                             }
                         }
-
-                        // compress to zip
-                        ZipFile.CreateFromDirectory(ZipFileTempPath, ZipFilePath, CompressionLevel.SmallestSize, false);
-
-                        // clean up
-                        if (Directory.Exists(ZipFileTempPath))
-                        {
-                            Directory.Delete(ZipFileTempPath, true);
-                        }
-
-                        // set completed
-                        dbDict["bs"] = CollectionItem.CollectionBuildStatus.Completed;
-                        db.ExecuteCMD(sql, dbDict);
                     }
-                    catch (Exception ex)
+
+                    // compress to zip
+                    Logging.Log(Logging.LogType.Information, "Collections", "Compressing collection");
+                    ZipFile.CreateFromDirectory(ZipFileTempPath, ZipFilePath, CompressionLevel.SmallestSize, false);
+
+                    // clean up
+                    if (Directory.Exists(ZipFileTempPath))
                     {
-                        // clean up
-                        if (Directory.Exists(ZipFileTempPath))
-                        {
-                            Directory.Delete(ZipFileTempPath, true);
-                        }
-
-                        if (File.Exists(ZipFilePath))
-                        {
-                            File.Delete(ZipFilePath);
-                        }
-
-                        // set failed
-                        dbDict["bs"] = CollectionItem.CollectionBuildStatus.Failed;
-                        db.ExecuteCMD(sql, dbDict);
-
-                        Logging.Log(Logging.LogType.Critical, "Collection Builder", "Collection building has failed", ex);
+                        Logging.Log(Logging.LogType.Information, "Collections", "Cleaning up");
+                        Directory.Delete(ZipFileTempPath, true);
                     }
+
+                    // set completed
+                    dbDict["bs"] = CollectionItem.CollectionBuildStatus.Completed;
+                    db.ExecuteCMD(sql, dbDict);
+                }
+                catch (Exception ex)
+                {
+                    // clean up
+                    if (Directory.Exists(ZipFileTempPath))
+                    {
+                        Directory.Delete(ZipFileTempPath, true);
+                    }
+
+                    if (File.Exists(ZipFilePath))
+                    {
+                        File.Delete(ZipFilePath);
+                    }
+
+                    // set failed
+                    dbDict["bs"] = CollectionItem.CollectionBuildStatus.Failed;
+                    db.ExecuteCMD(sql, dbDict);
+
+                    Logging.Log(Logging.LogType.Critical, "Collection Builder", "Collection building has failed", ex);
                 }
             }
         }
@@ -371,6 +538,7 @@ namespace gaseous_server.Classes
             string strPlayers = (string)Common.ReturnValueIfNull(row["Players"], "[ ]");
             string strPlayerPerspectives = (string)Common.ReturnValueIfNull(row["PlayerPerspectives"], "[ ]");
             string strThemes = (string)Common.ReturnValueIfNull(row["Themes"], "[ ]");
+            string strAlwaysInclude = (string)Common.ReturnValueIfNull(row["AlwaysInclude"], "[ ]");
 
             CollectionItem item = new CollectionItem();
             item.Id = (long)row["Id"];
@@ -386,6 +554,9 @@ namespace gaseous_server.Classes
             item.MaximumRomsPerPlatform = (int)Common.ReturnValueIfNull(row["MaximumRomsPerPlatform"], (int)-1);
             item.MaximumBytesPerPlatform = (long)Common.ReturnValueIfNull(row["MaximumBytesPerPlatform"], (long)-1);
             item.MaximumCollectionSizeInBytes = (long)Common.ReturnValueIfNull(row["MaximumCollectionSizeInBytes"], (long)-1);
+            item.FolderStructure = (CollectionItem.FolderStructures)(int)Common.ReturnValueIfNull(row["FolderStructure"], 0);
+            item.IncludeBIOSFiles = (bool)row["IncludeBIOSFiles"];
+            item.AlwaysInclude = Newtonsoft.Json.JsonConvert.DeserializeObject<List<CollectionItem.AlwaysIncludeItem>>(strAlwaysInclude);
             item.BuildStatus = (CollectionItem.CollectionBuildStatus)(int)Common.ReturnValueIfNull(row["BuiltStatus"], 0);
 
             return item;
@@ -411,6 +582,9 @@ namespace gaseous_server.Classes
             public int? MaximumRomsPerPlatform { get; set; }
             public long? MaximumBytesPerPlatform { get; set; }
             public long? MaximumCollectionSizeInBytes { get; set; }
+            public FolderStructures FolderStructure { get; set; } = FolderStructures.Gaseous;
+            public bool IncludeBIOSFiles { get; set; } = true;
+            public List<AlwaysIncludeItem> AlwaysInclude { get; set; }
 
             [JsonIgnore]
             public CollectionBuildStatus BuildStatus
@@ -473,6 +647,26 @@ namespace gaseous_server.Classes
                 Completed = 3,
                 Failed = 4
             }
+
+            public enum FolderStructures
+            {
+                Gaseous = 0,
+                RetroPie = 1
+            }
+
+            public class AlwaysIncludeItem
+            {
+                public long PlatformId { get; set; }
+                public long GameId { get; set; }
+                public AlwaysIncludeStatus InclusionState { get; set; }
+            }
+
+            public enum AlwaysIncludeStatus
+            {
+                None = 0,
+                AlwaysInclude = 1,
+                AlwaysExclude = 2
+            }
         }
 
         public class CollectionContents {
@@ -588,6 +782,8 @@ namespace gaseous_server.Classes
                     public string Slug { get; set; }
                     public long Cover { get; set;}
 
+                    public CollectionItem.AlwaysIncludeItem InclusionStatus { get; set; }
+
                     public List<Roms.GameRomItem> Roms { get; set; }
 
                     public long RomSize {

gaseous-server/Classes/ImportGames.cs

@@ -5,6 +5,7 @@ using System.Security.Policy;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using gaseous_tools;
+using IGDB.Models;
 using MySqlX.XDevAPI;
 using Org.BouncyCastle.Utilities.IO.Pem;
 using static gaseous_server.Classes.Metadata.Games;
@@ -22,7 +23,7 @@ namespace gaseous_server.Classes
 
 				// import files first
 				foreach (string importContent in importContents_Files) {
-					ImportGame.ImportGameFile(importContent);
+					ImportGame.ImportGameFile(importContent, null, false);
 				}
             }
 			else
@@ -37,23 +38,18 @@ namespace gaseous_server.Classes
 
 	public class ImportGame
 	{
-        public static void ImportGameFile(string GameFileImportPath, bool IsDirectory = false, bool ForceImport = false)
+        public static void ImportGameFile(string GameFileImportPath, IGDB.Models.Platform? OverridePlatform, bool IsDirectory = false)
 		{
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 			string sql = "";
 			Dictionary<string, object> dbDict = new Dictionary<string, object>();
 
-			string[] SkippableFiles = {
-				".DS_STORE",
-				"desktop.ini"
-			};
-            if (SkippableFiles.Contains<string>(Path.GetFileName(GameFileImportPath), StringComparer.OrdinalIgnoreCase))
+            if (Common.SkippableFiles.Contains<string>(Path.GetFileName(GameFileImportPath), StringComparer.OrdinalIgnoreCase))
 			{ 
                 Logging.Log(Logging.LogType.Debug, "Import Game", "Skipping item " + GameFileImportPath);
             }
 			else
 			{
-				//Logging.Log(Logging.LogType.Information, "Import Game", "Processing item " + GameFileImportPath);
 				if (IsDirectory == false)
 				{
 					FileInfo fi = new FileInfo(GameFileImportPath);
@@ -83,10 +79,20 @@ namespace gaseous_server.Classes
                             Models.Signatures_Games discoveredSignature = GetFileSignature(hash, fi, GameFileImportPath);
 
                             // get discovered platform
-                            IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(discoveredSignature.Flags.IGDBPlatformId);
-                            if (determinedPlatform == null)
+                            IGDB.Models.Platform? determinedPlatform = null;
+                            if (OverridePlatform == null)
                             {
-                                determinedPlatform = new IGDB.Models.Platform();
+                                determinedPlatform = Metadata.Platforms.GetPlatform(discoveredSignature.Flags.IGDBPlatformId);
+                                if (determinedPlatform == null)
+                                {
+                                    determinedPlatform = new IGDB.Models.Platform();
+                                }
+                            }
+                            else
+                            {
+                                determinedPlatform = OverridePlatform;
+                                discoveredSignature.Flags.IGDBPlatformId = (long)determinedPlatform.Id;
+                                discoveredSignature.Flags.IGDBPlatformName = determinedPlatform.Name;
                             }
 
                             IGDB.Models.Game determinedGame = SearchForGame(discoveredSignature.Game.Name, discoveredSignature.Flags.IGDBPlatformId);
@@ -144,6 +150,17 @@ namespace gaseous_server.Classes
 
                     if (zDiscoveredSignature.Score > discoveredSignature.Score)
                     {
+                        if (
+                            zDiscoveredSignature.Rom.SignatureSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.MAMEArcade || 
+                            zDiscoveredSignature.Rom.SignatureSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.MAMEMess
+                        )
+                        {
+                            zDiscoveredSignature.Rom.Name = zDiscoveredSignature.Game.Description + ".zip";
+                        }
+                        zDiscoveredSignature.Rom.Crc = discoveredSignature.Rom.Crc;
+                        zDiscoveredSignature.Rom.Md5 = discoveredSignature.Rom.Md5;
+                        zDiscoveredSignature.Rom.Sha1 = discoveredSignature.Rom.Sha1;
+                        zDiscoveredSignature.Rom.Size = discoveredSignature.Rom.Size;
                         discoveredSignature = zDiscoveredSignature;
 
                         break;
@@ -238,7 +255,7 @@ namespace gaseous_server.Classes
                     ri.Md5 = hash.md5hash;
                     ri.Sha1 = hash.sha1hash;
                     ri.Size = fi.Length;
-                    ri.SignatureSource = Models.Signatures_Games.RomItem.SignatureSourceType.None;
+                    ri.SignatureSource = gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None;
                 }
             }
 
@@ -275,6 +292,17 @@ namespace gaseous_server.Classes
                     else if (games.Length > 0)
                     {
                         Logging.Log(Logging.LogType.Information, "Import Game", "  " + games.Length + " search results found");
+
+                        // quite likely we've found sequels and alternate types
+                        foreach (Game game in games) {
+                            if (game.Name == SearchCandidate) {
+                                // found game title matches the search candidate
+                                determinedGame = Metadata.Games.GetGame((long)games[0].Id, false, false, false);
+                                Logging.Log(Logging.LogType.Information, "Import Game", "Found exact match!");
+                                GameFound = true;
+                                break;
+                            }
+                        }
                     }
                     else
                     {
@@ -340,11 +368,17 @@ namespace gaseous_server.Classes
             GameName = Regex.Replace(GameName, @"v(\d+\.)?(\d+\.)?(\*|\d+)$", "").Trim();
             GameName = Regex.Replace(GameName, @"Rev (\d+\.)?(\d+\.)?(\*|\d+)$", "").Trim();
 
+            // assumption: no games have () in their titles so we'll remove them
+            int idx = GameName.IndexOf('(');
+            if (idx >= 0) {
+                GameName = GameName.Substring(0, idx);
+            }
+
             List<string> SearchCandidates = new List<string>();
-            SearchCandidates.Add(GameName);
+            SearchCandidates.Add(GameName.Trim());
             if (GameName.Contains(" - "))
             {
-                SearchCandidates.Add(GameName.Replace(" - ", ": "));
+                SearchCandidates.Add(GameName.Replace(" - ", ": ").Trim());
                 SearchCandidates.Add(GameName.Substring(0, GameName.IndexOf(" - ")).Trim());
             }
             if (GameName.Contains(": "))
@@ -367,10 +401,10 @@ namespace gaseous_server.Classes
 
             if (UpdateId == 0)
             {
-                sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @path, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
+                sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource, MetadataGameName, MetadataVersion) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @Attributes, @romtype, @romtypemedia, @medialabel, @path, @metadatasource, @metadatagamename, @metadataversion); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
             } else
             {
-                sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Flags=@flags, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource WHERE Id=@id;";
+                sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;";
                 dbDict.Add("id", UpdateId);
             }
             dbDict.Add("platformid", Common.ReturnValueIfNull(determinedPlatform.Id, 0));
@@ -382,21 +416,23 @@ namespace gaseous_server.Classes
             dbDict.Add("crc", Common.ReturnValueIfNull(discoveredSignature.Rom.Crc, ""));
             dbDict.Add("developmentstatus", Common.ReturnValueIfNull(discoveredSignature.Rom.DevelopmentStatus, ""));
             dbDict.Add("metadatasource", discoveredSignature.Rom.SignatureSource);
+            dbDict.Add("metadatagamename", discoveredSignature.Game.Name);
+            dbDict.Add("metadataversion", 2);
 
-            if (discoveredSignature.Rom.flags != null)
+            if (discoveredSignature.Rom.Attributes != null)
             {
-                if (discoveredSignature.Rom.flags.Count > 0)
+                if (discoveredSignature.Rom.Attributes.Count > 0)
                 {
-                    dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(discoveredSignature.Rom.flags));
+                    dbDict.Add("attributes", Newtonsoft.Json.JsonConvert.SerializeObject(discoveredSignature.Rom.Attributes));
                 }
                 else
                 {
-                    dbDict.Add("flags", "[ ]");
+                    dbDict.Add("attributes", "[ ]");
                 }
             }
             else
             {
-                dbDict.Add("flags", "[ ]");
+                dbDict.Add("attributes", "[ ]");
             }
             dbDict.Add("romtype", (int)discoveredSignature.Rom.RomType);
             dbDict.Add("romtypemedia", Common.ReturnValueIfNull(discoveredSignature.Rom.RomTypeMedia, ""));
@@ -572,43 +608,46 @@ namespace gaseous_server.Classes
             string[] LibraryFiles = Directory.GetFiles(Config.LibraryConfiguration.LibraryDataDirectory, "*.*", SearchOption.AllDirectories);
             foreach (string LibraryFile in LibraryFiles)
             {
-                Common.hashObject LibraryFileHash = new Common.hashObject(LibraryFile);
-
-                // check if file is in database
-                bool romFound = false;
-                for (var i = 0; i < dtRoms.Rows.Count; i++)
+                if (!Common.SkippableFiles.Contains<string>(Path.GetFileName(LibraryFile), StringComparer.OrdinalIgnoreCase))
                 {
-                    long romId = (long)dtRoms.Rows[i]["Id"];
-                    string romPath = (string)dtRoms.Rows[i]["Path"];
-                    string romMd5 = (string)dtRoms.Rows[i]["MD5"];
+                    Common.hashObject LibraryFileHash = new Common.hashObject(LibraryFile);
 
-                    if ((LibraryFile == romPath) || (LibraryFileHash.md5hash == romMd5))
+                    // check if file is in database
+                    bool romFound = false;
+                    for (var i = 0; i < dtRoms.Rows.Count; i++)
                     {
-                        romFound = true;
-                        break;
-                    }
-                }
+                        long romId = (long)dtRoms.Rows[i]["Id"];
+                        string romPath = (string)dtRoms.Rows[i]["Path"];
+                        string romMd5 = (string)dtRoms.Rows[i]["MD5"];
 
-                if (romFound == false)
-                {
-                    // file is not in database - process it
-                    Common.hashObject hash = new Common.hashObject(LibraryFile);
-                    FileInfo fi = new FileInfo(LibraryFile);
-
-                    Models.Signatures_Games sig = GetFileSignature(hash, fi, LibraryFile);
-
-                    Logging.Log(Logging.LogType.Information, "Library Scan", " Orphaned file found in library: " + LibraryFile);
-
-                    // get discovered platform
-                    IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
-                    if (determinedPlatform == null)
-                    {
-                        determinedPlatform = new IGDB.Models.Platform();
+                        if ((LibraryFile == romPath) || (LibraryFileHash.md5hash == romMd5))
+                        {
+                            romFound = true;
+                            break;
+                        }
                     }
 
-                    IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
+                    if (romFound == false)
+                    {
+                        // file is not in database - process it
+                        Common.hashObject hash = new Common.hashObject(LibraryFile);
+                        FileInfo fi = new FileInfo(LibraryFile);
 
-                    StoreROM(hash, determinedGame, determinedPlatform, sig, LibraryFile);
+                        Models.Signatures_Games sig = GetFileSignature(hash, fi, LibraryFile);
+
+                        Logging.Log(Logging.LogType.Information, "Library Scan", " Orphaned file found in library: " + LibraryFile);
+
+                        // get discovered platform
+                        IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
+                        if (determinedPlatform == null)
+                        {
+                            determinedPlatform = new IGDB.Models.Platform();
+                        }
+
+                        IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
+
+                        StoreROM(hash, determinedGame, determinedPlatform, sig, LibraryFile);
+                    }
                 }
             }
 
@@ -623,24 +662,27 @@ namespace gaseous_server.Classes
                 {
                     long romId = (long)dtRoms.Rows[i]["Id"];
                     string romPath = (string)dtRoms.Rows[i]["Path"];
-                    Classes.Roms.GameRomItem.SourceType romMetadataSource = (Classes.Roms.GameRomItem.SourceType)(int)dtRoms.Rows[i]["MetadataSource"];
+                    gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType romMetadataSource = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(int)dtRoms.Rows[i]["MetadataSource"];
 
                     Logging.Log(Logging.LogType.Information, "Library Scan", " Processing ROM at path " + romPath);
 
                     if (File.Exists(romPath))
                     {
                         // file exists, so lets check to make sure the signature was matched, and update if a signature can be found
-                        if (romMetadataSource == Roms.GameRomItem.SourceType.None)
+                        if (
+                            romMetadataSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None ||
+                            (int)dtRoms.Rows[i]["MetadataVersion"] == 1
+                        )
                         {
                             Common.hashObject hash = new Common.hashObject
                             {
-                                md5hash = "",
-                                sha1hash = ""
+                                md5hash = (string)dtRoms.Rows[i]["MD5"],
+                                sha1hash = (string)dtRoms.Rows[i]["SHA1"]
                             };
                             FileInfo fi = new FileInfo(romPath);
 
                             Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath);
-                            if (sig.Rom.SignatureSource != Models.Signatures_Games.RomItem.SignatureSourceType.None)
+                            if (sig.Rom.SignatureSource != gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None)
                             {
                                 Logging.Log(Logging.LogType.Information, "Library Scan", " Update signature found for " + romPath);
 

gaseous-server/Classes/Metadata/AgeRating.cs

@@ -77,10 +77,17 @@ namespace gaseous_server.Classes.Metadata
                     UpdateSubClasses(returnValue);
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    UpdateSubClasses(returnValue);
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<AgeRating>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<AgeRating>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/AgeRatingContentDescriptions.cs

@@ -75,9 +75,17 @@ namespace gaseous_server.Classes.Metadata
                     Storage.NewCacheValue(returnValue);
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<AgeRatingContentDescription>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<AgeRatingContentDescription>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/AlternativeNames.cs

@@ -75,9 +75,17 @@ namespace gaseous_server.Classes.Metadata
                     Storage.NewCacheValue(returnValue);
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<AlternativeName>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<AlternativeName>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/Artworks.cs

@@ -78,10 +78,17 @@ namespace gaseous_server.Classes.Metadata
                     forceImageDownload = true;
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause, LogoPath);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause, LogoPath);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<Artwork>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<Artwork>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/Collections.cs

@@ -75,9 +75,17 @@ namespace gaseous_server.Classes.Metadata
                     Storage.NewCacheValue(returnValue);
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<Collection>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<Collection>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/Company.cs

@@ -74,9 +74,16 @@ namespace gaseous_server.Classes.Metadata
                     UpdateSubClasses(returnValue);
                     break;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    if (returnValue != null) { Storage.NewCacheValue(returnValue, true); }
-                    UpdateSubClasses(returnValue);
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<Company>(returnValue, "id", (long)searchValue);
+                    }
                     break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<Company>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/CompanyLogos.cs

@@ -80,12 +80,17 @@ namespace gaseous_server.Classes.Metadata
                     }
                     break;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause, LogoPath);
-                    if (returnValue != null)
+                    try
                     {
+                        returnValue = await GetObjectFromServer(WhereClause, LogoPath);
                         Storage.NewCacheValue(returnValue, true);
                         forceImageDownload = true;
                     }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<CompanyLogo>(returnValue, "id", (long)searchValue);
+                    }
                     break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<CompanyLogo>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/Covers.cs

@@ -77,10 +77,18 @@ namespace gaseous_server.Classes.Metadata
                     forceImageDownload = true;
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause, LogoPath);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause, LogoPath);
+                        Storage.NewCacheValue(returnValue, true);
+                        forceImageDownload = true;
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<Cover>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<Cover>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/ExternalGames.cs

@@ -78,12 +78,17 @@ namespace gaseous_server.Classes.Metadata
                     }
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    if (returnValue != null)
+                    try
                     {
+                        returnValue = await GetObjectFromServer(WhereClause);
                         Storage.NewCacheValue(returnValue, true);
                     }
-                    break;  
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<ExternalGame>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<ExternalGame>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/Franchises.cs

@@ -75,9 +75,17 @@ namespace gaseous_server.Classes.Metadata
                     Storage.NewCacheValue(returnValue);
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<Franchise>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<Franchise>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/GameModes.cs

@@ -68,18 +68,23 @@ namespace gaseous_server.Classes.Metadata
             }
 
             GameMode returnValue = new GameMode();
-            bool forceImageDownload = false;
             switch (cacheStatus)
             {
                 case Storage.CacheStatus.NotPresent:
                     returnValue = await GetObjectFromServer(WhereClause);
                     Storage.NewCacheValue(returnValue);
-                    forceImageDownload = true;
                     break;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<GameMode>(returnValue, "id", (long)searchValue);
+                    }
                     break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<GameMode>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/GameVideos.cs

@@ -68,19 +68,24 @@ namespace gaseous_server.Classes.Metadata
             }
 
             GameVideo returnValue = new GameVideo();
-            bool forceImageDownload = false;
             switch (cacheStatus)
             {
                 case Storage.CacheStatus.NotPresent:
                     returnValue = await GetObjectFromServer(WhereClause);
                     Storage.NewCacheValue(returnValue);
-                    forceImageDownload = true;
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<GameVideo>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<GameVideo>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/Games.cs

@@ -102,9 +102,16 @@ namespace gaseous_server.Classes.Metadata
                     UpdateSubClasses(returnValue, getAllMetadata, followSubGames);
                     return returnValue;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    UpdateSubClasses(returnValue, getAllMetadata, followSubGames);
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<Game>(returnValue, "id", (long)searchValue);
+                    }
                     return returnValue;
                 case Storage.CacheStatus.Current:
                     return Storage.GetCacheValue<Game>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/Genres.cs

@@ -68,19 +68,24 @@ namespace gaseous_server.Classes.Metadata
             }
 
             Genre returnValue = new Genre();
-            bool forceImageDownload = false;
             switch (cacheStatus)
             {
                 case Storage.CacheStatus.NotPresent:
                     returnValue = await GetObjectFromServer(WhereClause);
                     Storage.NewCacheValue(returnValue);
-                    forceImageDownload = true;
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<Genre>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<Genre>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/InvolvedCompany.cs

@@ -74,9 +74,16 @@ namespace gaseous_server.Classes.Metadata
                     UpdateSubClasses(returnValue);
                     break;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    UpdateSubClasses(returnValue);
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<InvolvedCompany>(returnValue, "id", (long)searchValue);
+                    }
                     break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<InvolvedCompany>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/MultiplayerModes.cs

@@ -68,18 +68,23 @@ namespace gaseous_server.Classes.Metadata
             }
 
             MultiplayerMode returnValue = new MultiplayerMode();
-            bool forceImageDownload = false;
             switch (cacheStatus)
             {
                 case Storage.CacheStatus.NotPresent:
                     returnValue = await GetObjectFromServer(WhereClause);
                     Storage.NewCacheValue(returnValue);
-                    forceImageDownload = true;
                     break;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<MultiplayerMode>(returnValue, "id", (long)searchValue);
+                    }
                     break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<MultiplayerMode>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/PlatformLogos.cs

@@ -80,12 +80,17 @@ namespace gaseous_server.Classes.Metadata
                     }
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause, LogoPath);
-                    if (returnValue != null)
+                    try
                     {
+                        returnValue = await GetObjectFromServer(WhereClause, LogoPath);
                         Storage.NewCacheValue(returnValue, true);
                         forceImageDownload = true;
                     }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<PlatformLogo>(returnValue, "id", (long)searchValue);
+                    }
                     break;  
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<PlatformLogo>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/PlatformVersions.cs

@@ -78,12 +78,17 @@ namespace gaseous_server.Classes.Metadata
                     }
                     return returnValue;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    if (returnValue != null)
+                    try
                     {
+                        returnValue = await GetObjectFromServer(WhereClause);
                         Storage.NewCacheValue(returnValue, true);
                         UpdateSubClasses(ParentPlatform, returnValue);
                     }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<PlatformVersion>(returnValue, "id", (long)searchValue);
+                    }
                     return returnValue;
                 case Storage.CacheStatus.Current:
                     return Storage.GetCacheValue<PlatformVersion>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/Platforms.cs

@@ -22,7 +22,7 @@ namespace gaseous_server.Classes.Metadata
                     Config.IGDB.Secret
                 );
 
-        public static Platform? GetPlatform(long Id)
+        public static Platform? GetPlatform(long Id, bool forceRefresh = false)
 		{
             if (Id == 0)
             {
@@ -46,18 +46,18 @@ namespace gaseous_server.Classes.Metadata
             }
             else
             {
-                Task<Platform> RetVal = _GetPlatform(SearchUsing.id, Id);
+                Task<Platform> RetVal = _GetPlatform(SearchUsing.id, Id, forceRefresh);
                 return RetVal.Result;
             }
         }
 
-		public static Platform GetPlatform(string Slug)
+		public static Platform GetPlatform(string Slug, bool forceRefresh = false)
 		{
-			Task<Platform> RetVal = _GetPlatform(SearchUsing.slug, Slug);
+			Task<Platform> RetVal = _GetPlatform(SearchUsing.slug, Slug, forceRefresh);
 			return RetVal.Result;
 		}
 
-		private static async Task<Platform> _GetPlatform(SearchUsing searchUsing, object searchValue)
+		private static async Task<Platform> _GetPlatform(SearchUsing searchUsing, object searchValue, bool forceRefresh)
 		{
             // check database first
             Storage.CacheStatus? cacheStatus = new Storage.CacheStatus();
@@ -70,6 +70,11 @@ namespace gaseous_server.Classes.Metadata
                 cacheStatus = Storage.GetCacheStatus("Platform", (string)searchValue);
             }
 
+            if (forceRefresh == true)
+            {
+                if (cacheStatus == Storage.CacheStatus.Current) { cacheStatus = Storage.CacheStatus.Expired; }
+            }
+
             // set up where clause
             string WhereClause = "";
             switch (searchUsing)
@@ -91,12 +96,22 @@ namespace gaseous_server.Classes.Metadata
                     returnValue = await GetObjectFromServer(WhereClause);
                     Storage.NewCacheValue(returnValue);
                     UpdateSubClasses(returnValue);
+                    AddPlatformMapping(returnValue);
                     return returnValue;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    UpdateSubClasses(returnValue);
-                    return returnValue;
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                        UpdateSubClasses(returnValue);
+                        AddPlatformMapping(returnValue);
+                        return returnValue;
+                    }
+                    catch (Exception ex)
+                    {
+                        Logging.Log(Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        return Storage.GetCacheValue<Platform>(returnValue, "id", (long)searchValue);
+                    }
                 case Storage.CacheStatus.Current:
                     return Storage.GetCacheValue<Platform>(returnValue, "id", (long)searchValue);
                 default:
@@ -120,6 +135,31 @@ namespace gaseous_server.Classes.Metadata
             }
         }
 
+        private static void AddPlatformMapping(Platform platform)
+        {
+            // ensure a mapping item exists for this platform
+            Models.PlatformMapping.PlatformMapItem item = new Models.PlatformMapping.PlatformMapItem();
+            try
+            {
+                Logging.Log(Logging.LogType.Information, "Platform Map", "Checking if " + platform.Name + " is in database.");
+                item = Models.PlatformMapping.GetPlatformMap((long)platform.Id);
+                // exists - skip
+                Logging.Log(Logging.LogType.Information, "Platform Map", "Skipping import of " + platform.Name + " - already in database.");
+            }
+            catch
+            {
+                Logging.Log(Logging.LogType.Information, "Platform Map", "Importing " + platform.Name + " from predefined data.");
+                // doesn't exist - add it
+                item = new Models.PlatformMapping.PlatformMapItem{
+                    IGDBId = (long)platform.Id,
+                    IGDBName = platform.Name,
+                    IGDBSlug = platform.Slug,
+                    AlternateNames = new List<string>{ platform.AlternativeName }
+                };
+                Models.PlatformMapping.WritePlatformMap(item, false, true);
+            }
+        }
+
         private enum SearchUsing
         {
             id,

gaseous-server/Classes/Metadata/PlayerPerspectives.cs

@@ -77,9 +77,16 @@ namespace gaseous_server.Classes.Metadata
                     forceImageDownload = true;
                     break;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<PlayerPerspective>(returnValue, "id", (long)searchValue);
+                    }
                     break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<PlayerPerspective>(returnValue, "id", (long)searchValue);

gaseous-server/Classes/Metadata/Screenshots.cs

@@ -78,10 +78,18 @@ namespace gaseous_server.Classes.Metadata
                     forceImageDownload = true;
                     break;  
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause, LogoPath);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
-                    break;  
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause, LogoPath);
+                        Storage.NewCacheValue(returnValue, true);
+                        forceImageDownload = true;
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        returnValue = Storage.GetCacheValue<Screenshot>(returnValue, "id", (long)searchValue);
+                    }
+                    break;
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<Screenshot>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/Metadata/Themes.cs

@@ -77,10 +77,17 @@ namespace gaseous_server.Classes.Metadata
                     forceImageDownload = true;
                     break;
                 case Storage.CacheStatus.Expired:
-                    returnValue = await GetObjectFromServer(WhereClause);
-                    Storage.NewCacheValue(returnValue, true);
-                    forceImageDownload = true;
-                    break;
+                    try
+                    {
+                        returnValue = await GetObjectFromServer(WhereClause);
+                        Storage.NewCacheValue(returnValue, true);
+                        return returnValue;
+                    }
+                    catch (Exception ex)
+                    {
+                        gaseous_tools.Logging.Log(gaseous_tools.Logging.LogType.Warning, "Metadata: " + returnValue.GetType().Name, "An error occurred while connecting to IGDB. WhereClause: " + WhereClause, ex);
+                        return Storage.GetCacheValue<Theme>(returnValue, "id", (long)searchValue);
+                    }
                 case Storage.CacheStatus.Current:
                     returnValue = Storage.GetCacheValue<Theme>(returnValue, "id", (long)searchValue);
                     break;

gaseous-server/Classes/MetadataManagement.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Data;
+using gaseous_server.Models;
 using gaseous_tools;
 
 namespace gaseous_server.Classes
@@ -9,8 +10,29 @@ namespace gaseous_server.Classes
 		public static void RefreshMetadata(bool forceRefresh = false)
 		{
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-            string sql = "SELECT Id, `Name` FROM Game;";
-			DataTable dt = db.ExecuteCMD(sql);
+            string sql = "";
+			DataTable dt = new DataTable();
+			
+			// update platforms
+			sql = "SELECT Id, `Name` FROM Platform;";
+			dt = db.ExecuteCMD(sql);
+
+			foreach (DataRow dr in dt.Rows)
+			{
+				try
+				{
+					Logging.Log(Logging.LogType.Information, "Metadata Refresh", "Refreshing metadata for platform " + dr["name"] + " (" + dr["id"] + ")");
+					Metadata.Platforms.GetPlatform((long)dr["id"], true);
+				}
+				catch (Exception ex)
+				{
+					Logging.Log(Logging.LogType.Critical, "Metadata Refresh", "An error occurred while refreshing metadata for " + dr["name"], ex);
+				}
+			}
+
+			// update games
+			sql = "SELECT Id, `Name` FROM Game;";
+			dt = db.ExecuteCMD(sql);
 
 			foreach (DataRow dr in dt.Rows)
 			{

gaseous-server/Classes/Roms.cs

@@ -95,7 +95,7 @@ namespace gaseous_server.Classes
 
 		private static GameRomItem BuildRom(DataRow romDR)
 		{
-            GameRomItem romItem = new GameRomItem
+			GameRomItem romItem = new GameRomItem
             {
                 Id = (long)romDR["id"],
                 PlatformId = (long)romDR["platformid"],
@@ -103,16 +103,17 @@ namespace gaseous_server.Classes
                 GameId = (long)romDR["gameid"],
                 Name = (string)romDR["name"],
                 Size = (long)romDR["size"],
-                CRC = (string)romDR["crc"],
-                MD5 = (string)romDR["md5"],
-                SHA1 = (string)romDR["sha1"],
+                CRC = ((string)romDR["crc"]).ToLower(),
+                MD5 = ((string)romDR["md5"]).ToLower(),
+                SHA1 = ((string)romDR["sha1"]).ToLower(),
                 DevelopmentStatus = (string)romDR["developmentstatus"],
-                Flags = Newtonsoft.Json.JsonConvert.DeserializeObject<string[]>((string)romDR["flags"]),
+                Attributes = Newtonsoft.Json.JsonConvert.DeserializeObject<List<KeyValuePair<string, object>>>((string)Common.ReturnValueIfNull(romDR["attributes"], "[ ]")),
                 RomType = (int)romDR["romtype"],
                 RomTypeMedia = (string)romDR["romtypemedia"],
                 MediaLabel = (string)romDR["medialabel"],
                 Path = (string)romDR["path"],
-				Source = (GameRomItem.SourceType)(Int32)romDR["metadatasource"]
+				Source = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(Int32)romDR["metadatasource"],
+				SignatureSourceGameTitle = (string)Common.ReturnValueIfNull(romDR["MetadataGameName"], "")
             };
 
 			// check for a web emulator and update the romItem
@@ -145,17 +146,13 @@ namespace gaseous_server.Classes
 			public string? SHA1 { get; set; }
 			public string? DevelopmentStatus { get; set; }
 			public string[]? Flags { get; set; }
+			public List<KeyValuePair<string, object>>? Attributes { get; set;}
 			public int RomType { get; set; }
 			public string? RomTypeMedia { get; set; }
 			public string? MediaLabel { get; set; }
 			public string? Path { get; set; }
-            public SourceType Source { get; set; }
-
-            public enum SourceType
-            {
-                None = 0,
-                TOSEC = 1
-            }
+            public gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType Source { get; set; }
+			public string? SignatureSourceGameTitle { get; set;}
         }
     }
 }

gaseous-server/Classes/SignatureIngestors/TOSEC.cs

@@ -1,234 +0,0 @@
-using System;
-using System.IO;
-using MySql.Data.MySqlClient;
-using gaseous_romsignatureobject;
-using gaseous_signature_parser.parsers;
-using gaseous_tools;
-using MySqlX.XDevAPI;
-
-namespace gaseous_server.SignatureIngestors.TOSEC
-{
-    public class TOSECIngestor
-    {
-        public void Import(string SearchPath)
-        {
-            // connect to database
-            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-
-            // process provided files
-            Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing from " + SearchPath);
-            if (Directory.Exists(Config.LibraryConfiguration.LibrarySignatureImportDirectory_TOSEC))
-            {
-                string[] tosecPathContents = Directory.GetFiles(SearchPath);
-                Array.Sort(tosecPathContents);
-
-                string sql = "";
-                Dictionary<string, object> dbDict = new Dictionary<string, object>();
-                System.Data.DataTable sigDB;
-
-                for (UInt16 i = 0; i < tosecPathContents.Length; ++i)
-                {
-                    string tosecXMLFile = tosecPathContents[i];
-
-                    // check tosec file md5
-                    Common.hashObject hashObject = new Common.hashObject(tosecXMLFile);
-                    sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
-                    dbDict = new Dictionary<string, object>();
-                    dbDict.Add("sourcemd5", hashObject.md5hash);
-                    sigDB = db.ExecuteCMD(sql, dbDict);
-
-                    if (sigDB.Rows.Count == 0)
-                    {
-                        try
-                        {
-                            Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing file: " + tosecXMLFile);
-
-                            // start parsing file
-                            TosecParser tosecParser = new TosecParser();
-                            RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile);
-
-                            // store in database
-
-                            // store source object
-                            bool processGames = false;
-                            if (tosecObject.SourceMd5 != null)
-                            {
-                                sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
-                                dbDict = new Dictionary<string, object>();
-                                dbDict.Add("name", Common.ReturnValueIfNull(tosecObject.Name, ""));
-                                dbDict.Add("description", Common.ReturnValueIfNull(tosecObject.Description, ""));
-                                dbDict.Add("category", Common.ReturnValueIfNull(tosecObject.Category, ""));
-                                dbDict.Add("version", Common.ReturnValueIfNull(tosecObject.Version, ""));
-                                dbDict.Add("author", Common.ReturnValueIfNull(tosecObject.Author, ""));
-                                dbDict.Add("email", Common.ReturnValueIfNull(tosecObject.Email, ""));
-                                dbDict.Add("homepage", Common.ReturnValueIfNull(tosecObject.Homepage, ""));
-                                dbDict.Add("uri", Common.ReturnValueIfNull(tosecObject.Url, ""));
-                                dbDict.Add("sourcetype", Common.ReturnValueIfNull(tosecObject.SourceType, ""));
-                                dbDict.Add("sourcemd5", tosecObject.SourceMd5);
-                                dbDict.Add("sourcesha1", tosecObject.SourceSHA1);
-
-                                sigDB = db.ExecuteCMD(sql, dbDict);
-                                if (sigDB.Rows.Count == 0)
-                                {
-                                    // entry not present, insert it
-                                    sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)";
-
-                                    db.ExecuteCMD(sql, dbDict);
-
-                                    processGames = true;
-                                }
-
-                                if (processGames == true)
-                                {
-                                    for (int x = 0; x < tosecObject.Games.Count; ++x)
-                                    {
-                                        RomSignatureObject.Game gameObject = tosecObject.Games[x];
-
-                                        // set up game dictionary
-                                        dbDict = new Dictionary<string, object>();
-                                        dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, ""));
-                                        dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, ""));
-                                        dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, ""));
-                                        dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, ""));
-                                        dbDict.Add("demo", (int)gameObject.Demo);
-                                        dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, ""));
-                                        dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, ""));
-                                        dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, ""));
-                                        dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, ""));
-                                        dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, ""));
-                                        dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, ""));
-                                        dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, ""));
-
-                                        // store platform
-                                        int gameSystem = 0;
-                                        if (gameObject.System != null)
-                                        {
-                                            sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform";
-
-                                            sigDB = db.ExecuteCMD(sql, dbDict);
-                                            if (sigDB.Rows.Count == 0)
-                                            {
-                                                // entry not present, insert it
-                                                sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                                                sigDB = db.ExecuteCMD(sql, dbDict);
-
-                                                gameSystem = Convert.ToInt32(sigDB.Rows[0][0]);
-                                            }
-                                            else
-                                            {
-                                                gameSystem = (int)sigDB.Rows[0][0];
-                                            }
-                                        }
-                                        dbDict.Add("systemid", gameSystem);
-
-                                        // store publisher
-                                        int gamePublisher = 0;
-                                        if (gameObject.Publisher != null)
-                                        {
-                                            sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher";
-
-                                            sigDB = db.ExecuteCMD(sql, dbDict);
-                                            if (sigDB.Rows.Count == 0)
-                                            {
-                                                // entry not present, insert it
-                                                sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                                                sigDB = db.ExecuteCMD(sql, dbDict);
-                                                gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]);
-                                            }
-                                            else
-                                            {
-                                                gamePublisher = (int)sigDB.Rows[0][0];
-                                            }
-                                        }
-                                        dbDict.Add("publisherid", gamePublisher);
-
-                                        // store game
-                                        int gameId = 0;
-                                        sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language";
-
-                                        sigDB = db.ExecuteCMD(sql, dbDict);
-                                        if (sigDB.Rows.Count == 0)
-                                        {
-                                            // entry not present, insert it
-                                            sql = "INSERT INTO Signatures_Games " +
-                                                "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
-                                                "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                                            sigDB = db.ExecuteCMD(sql, dbDict);
-
-                                            gameId = Convert.ToInt32(sigDB.Rows[0][0]);
-                                        }
-                                        else
-                                        {
-                                            gameId = (int)sigDB.Rows[0][0];
-                                        }
-
-                                        // store rom
-                                        foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
-                                        {
-                                            if (romObject.Md5 != null)
-                                            {
-                                                int romId = 0;
-                                                sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5";
-                                                dbDict = new Dictionary<string, object>();
-                                                dbDict.Add("gameid", gameId);
-                                                dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, ""));
-                                                dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, ""));
-                                                dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, ""));
-                                                dbDict.Add("md5", romObject.Md5);
-                                                dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, ""));
-                                                dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, ""));
-
-                                                if (romObject.flags != null)
-                                                {
-                                                    if (romObject.flags.Count > 0)
-                                                    {
-                                                        dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags));
-                                                    }
-                                                    else
-                                                    {
-                                                        dbDict.Add("flags", "[ ]");
-                                                    }
-                                                }
-                                                else
-                                                {
-                                                    dbDict.Add("flags", "[ ]");
-                                                }
-                                                dbDict.Add("romtype", (int)romObject.RomType);
-                                                dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, ""));
-                                                dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
-                                                dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC);
-
-                                                sigDB = db.ExecuteCMD(sql, dbDict);
-                                                if (sigDB.Rows.Count == 0)
-                                                {
-                                                    // entry not present, insert it
-                                                    sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                                                    sigDB = db.ExecuteCMD(sql, dbDict);
-
-
-                                                    romId = Convert.ToInt32(sigDB.Rows[0][0]);
-                                                }
-                                                else
-                                                {
-                                                    romId = (int)sigDB.Rows[0][0];
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                        catch (Exception ex)
-                        {
-                            Logging.Log(Logging.LogType.Warning, "Signature Ingestor - TOSEC", "Invalid import file: " + tosecXMLFile, ex);
-                        }
-                    }
-                    else
-                    {
-                        Logging.Log(Logging.LogType.Debug, "Signature Ingestor - TOSEC", "Rejecting already imported file: " + tosecXMLFile);
-                    }
-                }
-            }
-        }
-    }
-}

gaseous-server/Classes/SignatureIngestors/XML.cs

@@ -0,0 +1,248 @@
+using System;
+using System.IO;
+using MySql.Data.MySqlClient;
+using gaseous_signature_parser.models.RomSignatureObject;
+using gaseous_tools;
+using MySqlX.XDevAPI;
+using System.Data;
+
+namespace gaseous_server.SignatureIngestors.XML
+{
+    public class XMLIngestor
+    {
+        public void Import(string SearchPath, gaseous_signature_parser.parser.SignatureParser XMLType)
+        {
+            // connect to database
+            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+
+            // process provided files
+            Logging.Log(Logging.LogType.Information, "Signature Ingestor - XML", "Importing from " + SearchPath);
+            if (!Directory.Exists(SearchPath))
+            {
+                Directory.CreateDirectory(SearchPath);
+            }
+
+            string[] PathContents = Directory.GetFiles(SearchPath);
+            Array.Sort(PathContents);
+
+            string sql = "";
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+            System.Data.DataTable sigDB;
+
+            for (UInt16 i = 0; i < PathContents.Length; ++i)
+            {
+                string XMLFile = PathContents[i];
+
+                // check xml file md5
+                Common.hashObject hashObject = new Common.hashObject(XMLFile);
+                sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
+                dbDict = new Dictionary<string, object>();
+                dbDict.Add("sourcemd5", hashObject.md5hash);
+                sigDB = db.ExecuteCMD(sql, dbDict);
+
+                if (sigDB.Rows.Count == 0)
+                {
+                    try
+                    {
+                        Logging.Log(Logging.LogType.Information, "Signature Ingestor - XML", "Importing file: " + XMLFile);
+
+                        // start parsing file
+                        gaseous_signature_parser.parser Parser = new gaseous_signature_parser.parser();
+                        RomSignatureObject Object = Parser.ParseSignatureDAT(XMLFile, XMLType);
+
+                        // store in database
+                        string[] flipNameAndDescription = {
+                            "MAMEArcade",
+                            "MAMEMess"
+                        };
+
+                        // store source object
+                        bool processGames = false;
+                        if (Object.SourceMd5 != null)
+                        {
+                            sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
+                            dbDict = new Dictionary<string, object>();
+                            dbDict.Add("name", Common.ReturnValueIfNull(Object.Name, ""));
+                            dbDict.Add("description", Common.ReturnValueIfNull(Object.Description, ""));
+                            dbDict.Add("category", Common.ReturnValueIfNull(Object.Category, ""));
+                            dbDict.Add("version", Common.ReturnValueIfNull(Object.Version, ""));
+                            dbDict.Add("author", Common.ReturnValueIfNull(Object.Author, ""));
+                            dbDict.Add("email", Common.ReturnValueIfNull(Object.Email, ""));
+                            dbDict.Add("homepage", Common.ReturnValueIfNull(Object.Homepage, ""));
+                            dbDict.Add("uri", Common.ReturnValueIfNull(Object.Url, ""));
+                            dbDict.Add("sourcetype", Common.ReturnValueIfNull(Object.SourceType, ""));
+                            dbDict.Add("sourcemd5", Object.SourceMd5);
+                            dbDict.Add("sourcesha1", Object.SourceSHA1);
+
+                            sigDB = db.ExecuteCMD(sql, dbDict);
+                            if (sigDB.Rows.Count == 0)
+                            {
+                                // entry not present, insert it
+                                sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)";
+
+                                db.ExecuteCMD(sql, dbDict);
+
+                                processGames = true;
+                            }
+
+                            if (processGames == true)
+                            {
+                                for (int x = 0; x < Object.Games.Count; ++x)
+                                {
+                                    RomSignatureObject.Game gameObject = Object.Games[x];
+
+                                    // set up game dictionary
+                                    dbDict = new Dictionary<string, object>();
+                                    if (flipNameAndDescription.Contains(Object.SourceType)) 
+                                    {
+                                        dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Description, ""));
+                                        dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Name, ""));
+                                    }
+                                    else
+                                    {
+                                        dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, ""));
+                                        dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, ""));
+                                    }
+                                    dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, ""));
+                                    dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, ""));
+                                    dbDict.Add("demo", (int)gameObject.Demo);
+                                    dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, ""));
+                                    dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, ""));
+                                    dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, ""));
+                                    dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, ""));
+                                    dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, ""));
+                                    dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, ""));
+                                    dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, ""));
+
+                                    // store platform
+                                    int gameSystem = 0;
+                                    if (gameObject.System != null)
+                                    {
+                                        sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform";
+
+                                        sigDB = db.ExecuteCMD(sql, dbDict);
+                                        if (sigDB.Rows.Count == 0)
+                                        {
+                                            // entry not present, insert it
+                                            sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
+                                            sigDB = db.ExecuteCMD(sql, dbDict);
+
+                                            gameSystem = Convert.ToInt32(sigDB.Rows[0][0]);
+                                        }
+                                        else
+                                        {
+                                            gameSystem = (int)sigDB.Rows[0][0];
+                                        }
+                                    }
+                                    dbDict.Add("systemid", gameSystem);
+
+                                    // store publisher
+                                    int gamePublisher = 0;
+                                    if (gameObject.Publisher != null)
+                                    {
+                                        sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher";
+
+                                        sigDB = db.ExecuteCMD(sql, dbDict);
+                                        if (sigDB.Rows.Count == 0)
+                                        {
+                                            // entry not present, insert it
+                                            sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
+                                            sigDB = db.ExecuteCMD(sql, dbDict);
+                                            gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]);
+                                        }
+                                        else
+                                        {
+                                            gamePublisher = (int)sigDB.Rows[0][0];
+                                        }
+                                    }
+                                    dbDict.Add("publisherid", gamePublisher);
+
+                                    // store game
+                                    int gameId = 0;
+                                    sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language";
+
+                                    sigDB = db.ExecuteCMD(sql, dbDict);
+                                    if (sigDB.Rows.Count == 0)
+                                    {
+                                        // entry not present, insert it
+                                        sql = "INSERT INTO Signatures_Games " +
+                                            "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
+                                            "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
+                                        sigDB = db.ExecuteCMD(sql, dbDict);
+
+                                        gameId = Convert.ToInt32(sigDB.Rows[0][0]);
+                                    }
+                                    else
+                                    {
+                                        gameId = (int)sigDB.Rows[0][0];
+                                    }
+
+                                    // store rom
+                                    foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
+                                    {
+                                        if (romObject.Md5 != null || romObject.Sha1 != null)
+                                        {
+                                            int romId = 0;
+                                            sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5";
+                                            dbDict = new Dictionary<string, object>();
+                                            dbDict.Add("gameid", gameId);
+                                            dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, ""));
+                                            dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, ""));
+                                            dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, "").ToString().ToLower());
+                                            dbDict.Add("md5", Common.ReturnValueIfNull(romObject.Md5, "").ToString().ToLower());
+                                            dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, "").ToString().ToLower());
+                                            dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, ""));
+
+                                            if (romObject.Attributes != null)
+                                            {
+                                                if (romObject.Attributes.Count > 0)
+                                                {
+                                                    dbDict.Add("attributes", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.Attributes));
+                                                }
+                                                else
+                                                {
+                                                    dbDict.Add("attributes", "[ ]");
+                                                }
+                                            }
+                                            else
+                                            {
+                                                dbDict.Add("attributes", "[ ]");
+                                            }
+                                            dbDict.Add("romtype", (int)romObject.RomType);
+                                            dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, ""));
+                                            dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
+                                            dbDict.Add("metadatasource", romObject.SignatureSource);
+
+                                            sigDB = db.ExecuteCMD(sql, dbDict);
+                                            if (sigDB.Rows.Count == 0)
+                                            {
+                                                // entry not present, insert it
+                                                sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @attributes, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
+                                                sigDB = db.ExecuteCMD(sql, dbDict);
+
+
+                                                romId = Convert.ToInt32(sigDB.Rows[0][0]);
+                                            }
+                                            else
+                                            {
+                                                romId = (int)sigDB.Rows[0][0];
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        Logging.Log(Logging.LogType.Warning, "Signature Ingestor - XML", "Invalid import file: " + XMLFile, ex);
+                    }
+                }
+                else
+                {
+                    Logging.Log(Logging.LogType.Debug, "Signature Ingestor - XML", "Rejecting already imported file: " + XMLFile);
+                }
+            }
+        }
+    }
+}

gaseous-server/Controllers/CollectionsController.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.IO.Compression;
 using System.Linq;
 using System.Threading.Tasks;
+using gaseous_server.Classes;
 using Microsoft.AspNetCore.Mvc;
 
 namespace gaseous_server.Controllers
@@ -82,14 +83,14 @@ namespace gaseous_server.Controllers
         [ProducesResponseType(StatusCodes.Status404NotFound)]
         public ActionResult GetCollectionRomsPreview(Classes.Collections.CollectionItem Item)
         {
-            //try
-            //{
+            try
+            {
                 return Ok(Classes.Collections.GetCollectionContent(Item));
-            //}
-            //catch (Exception ex)
-            //{
-            //    return NotFound(ex);
-            //}
+            }
+            catch (Exception ex)
+            {
+               return NotFound(ex);
+            }
         }
 
         /// <summary>
@@ -159,7 +160,43 @@ namespace gaseous_server.Controllers
         {
             try
             {
-                return Ok(Classes.Collections.EditCollection(CollectionId, Item));
+                return Ok(Classes.Collections.EditCollection(CollectionId, Item, true));
+            }
+            catch
+            {
+                return NotFound();
+            }
+        }
+
+        /// <summary>
+        /// Edits an existing collection
+        /// </summary>
+        /// <param name="CollectionId"></param>
+        /// <param name="Item"></param>
+        /// <returns></returns>
+        [HttpPatch]
+        [Route("{CollectionId}/AlwaysInclude")]
+        [ProducesResponseType(typeof(Classes.Collections.CollectionItem), StatusCodes.Status200OK)]
+        [ProducesResponseType(StatusCodes.Status404NotFound)]
+        public ActionResult EditCollectionAlwaysInclude(long CollectionId, [FromQuery]bool Rebuild, [FromBody]Collections.CollectionItem.AlwaysIncludeItem Inclusion)
+        {
+            try
+            {
+                Collections.CollectionItem collectionItem = Classes.Collections.GetCollection(CollectionId);
+                bool ItemFound = false;
+                foreach (Collections.CollectionItem.AlwaysIncludeItem includeItem in collectionItem.AlwaysInclude)
+                {
+                    if (includeItem.PlatformId == Inclusion.PlatformId && includeItem.GameId == Inclusion.GameId)
+                    {
+                        ItemFound = true;
+                    }
+                }
+                if (ItemFound == false)
+                {
+                    collectionItem.AlwaysInclude.Add(Inclusion);
+                }
+
+                return Ok(Classes.Collections.EditCollection(CollectionId, collectionItem, Rebuild));
             }
             catch
             {

gaseous-server/Controllers/LogsController.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using gaseous_tools;
+using Microsoft.AspNetCore.Mvc;
+
+namespace gaseous_server.Controllers
+{
+    [ApiController]
+    [Route("api/v1/[controller]")]
+    public class LogsController : Controller
+    {
+        [HttpGet]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        public List<Logging.LogItem> Logs(long? StartIndex, int PageNumber = 1, int PageSize = 100)
+        {
+            return Logging.GetLogs(StartIndex, PageNumber, PageSize);
+        }
+    }
+}

gaseous-server/Controllers/PlatformMapsController.cs

@@ -0,0 +1,166 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Threading.Tasks;
+using gaseous_server.Classes.Metadata;
+using gaseous_server.Models;
+using gaseous_tools;
+using IGDB.Models;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.CodeAnalysis.Scripting;
+
+namespace gaseous_server.Controllers
+{
+    [Route("api/v1/[controller]")]
+    [ApiController]
+    public class PlatformMapsController : Controller
+    {
+        [HttpGet]
+        [ProducesResponseType(typeof(List<PlatformMapping.PlatformMapItem>), StatusCodes.Status200OK)]
+        public ActionResult GetPlatformMap(bool ResetToDefault = false)
+        {
+            if (ResetToDefault == true)
+            {
+                PlatformMapping.ExtractPlatformMap(true);
+            }
+
+            return Ok(PlatformMapping.PlatformMap);
+        }
+
+        [HttpGet]
+        [Route("{PlatformId}")]
+        [ProducesResponseType(typeof(PlatformMapping.PlatformMapItem), StatusCodes.Status200OK)]
+        [ProducesResponseType(StatusCodes.Status404NotFound)]
+        public ActionResult PlatformMap(long PlatformId)
+        {
+            try
+            {
+                PlatformMapping.PlatformMapItem platformMapItem = PlatformMapping.GetPlatformMap(PlatformId);
+
+                if (platformMapItem != null)
+                {
+                    return Ok(platformMapItem);
+                }
+                else
+                {
+                    return NotFound();
+                }
+            }
+            catch
+            {
+                return NotFound();
+            }
+        }
+
+        [HttpPost]
+        [ProducesResponseType(typeof(List<IFormFile>), StatusCodes.Status200OK)]
+        [RequestSizeLimit(long.MaxValue)]
+        [DisableRequestSizeLimit, RequestFormLimits(MultipartBodyLengthLimit = long.MaxValue, ValueLengthLimit = int.MaxValue)]
+        public async Task<IActionResult> UploadPlatformMap(List<IFormFile> files)
+        {
+            Guid sessionid = Guid.NewGuid();
+
+            string workPath = Path.Combine(Config.LibraryConfiguration.LibraryUploadDirectory, sessionid.ToString());
+
+            long size = files.Sum(f => f.Length);
+
+            List<Dictionary<string, object>> UploadedFiles = new List<Dictionary<string, object>>();
+
+            foreach (IFormFile formFile in files)
+            {
+                if (formFile.Length > 0)
+                {
+                    Guid FileId = Guid.NewGuid();
+
+                    string filePath = Path.Combine(workPath, Path.GetFileName(formFile.FileName));
+
+                    if (!Directory.Exists(workPath))
+                    {
+                        Directory.CreateDirectory(workPath);
+                    }
+
+                    using (var stream = System.IO.File.Create(filePath))
+                    {
+                        await formFile.CopyToAsync(stream);
+
+                        Dictionary<string, object> UploadedFile = new Dictionary<string, object>();
+                        UploadedFile.Add("id", FileId.ToString());
+                        UploadedFile.Add("originalname", Path.GetFileName(formFile.FileName));
+                        UploadedFile.Add("fullpath", filePath);
+                        UploadedFiles.Add(UploadedFile);
+                    }
+                }
+            }
+
+            // Process uploaded files
+            foreach (Dictionary<string, object> UploadedFile in UploadedFiles)
+            {
+                Models.PlatformMapping.ExtractPlatformMap((string)UploadedFile["fullpath"]);
+            }
+
+            if (Directory.Exists(workPath))
+            {
+                Directory.Delete(workPath, true);
+            }
+
+            return Ok(new { count = files.Count, size });
+        }
+
+        // [HttpPost]
+        // [Route("{PlatformId}")]
+        // [ProducesResponseType(typeof(PlatformMapping.PlatformMapItem), StatusCodes.Status200OK)]
+        // [ProducesResponseType(StatusCodes.Status404NotFound)]
+        // [ProducesResponseType(StatusCodes.Status409Conflict)]
+        // public ActionResult NewPlatformMap(long PlatformId, PlatformMapping.PlatformMapItem Map)
+        // {
+        //     try
+        //     {
+        //         PlatformMapping.PlatformMapItem platformMapItem = PlatformMapping.GetPlatformMap(PlatformId);
+
+        //         if (platformMapItem != null)
+        //         {
+        //             return Conflict();
+        //         }
+        //         else
+        //         {
+        //             PlatformMapping.WritePlatformMap(Map, false, false);
+        //             return Ok(PlatformMapping.GetPlatformMap(PlatformId));
+        //         }
+        //     }
+        //     catch
+        //     {
+        //         return NotFound();
+        //     }
+        // }
+
+        [HttpPatch]
+        [Route("{PlatformId}")]
+        [ProducesResponseType(typeof(PlatformMapping.PlatformMapItem), StatusCodes.Status200OK)]
+        [ProducesResponseType(StatusCodes.Status404NotFound)]
+        public ActionResult EditPlatformMap(long PlatformId, PlatformMapping.PlatformMapItem Map)
+        {
+            try
+            {
+                PlatformMapping.PlatformMapItem platformMapItem = PlatformMapping.GetPlatformMap(PlatformId);
+
+                if (platformMapItem != null)
+                {
+                    PlatformMapping.WritePlatformMap(Map, true, false);
+                    return Ok(PlatformMapping.GetPlatformMap(PlatformId));
+                }
+                else
+                {
+                    return NotFound();
+                }
+            }
+            catch
+            {
+                return NotFound();
+            }
+        }
+    }
+}

gaseous-server/Controllers/PlatformsController.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using System.Reflection;
 using System.Threading.Tasks;
 using gaseous_server.Classes.Metadata;
+using gaseous_server.Models;
 using gaseous_tools;
 using IGDB.Models;
 using Microsoft.AspNetCore.Http;
@@ -21,6 +22,11 @@ namespace gaseous_server.Controllers
         [HttpGet]
         [ProducesResponseType(typeof(List<Platform>), StatusCodes.Status200OK)]
         public ActionResult Platform()
+        {
+            return Ok(PlatformsController.GetPlatforms());
+        }
+
+        public static List<Platform> GetPlatforms()
         {
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
 
@@ -34,7 +40,7 @@ namespace gaseous_server.Controllers
                 RetVal.Add(Classes.Metadata.Platforms.GetPlatform((long)dr["id"]));
             }
 
-            return Ok(RetVal);
+            return RetVal;
         }
 
         [HttpGet]

gaseous-server/Controllers/RomsController.cs

@@ -25,7 +25,7 @@ namespace gaseous_server.Controllers
         [ProducesResponseType(typeof(List<IFormFile>), StatusCodes.Status200OK)]
         [RequestSizeLimit(long.MaxValue)]
         [DisableRequestSizeLimit, RequestFormLimits(MultipartBodyLengthLimit = long.MaxValue, ValueLengthLimit = int.MaxValue)]
-        public async Task<IActionResult> UploadRom(List<IFormFile> files)
+        public async Task<IActionResult> UploadRom(List<IFormFile> files, long? OverridePlatformId = null)
         {
             Guid sessionid = Guid.NewGuid();
 
@@ -61,12 +61,17 @@ namespace gaseous_server.Controllers
                 }
             }
 
-            // Process uploaded files
-            // Don't rely on or trust the FileName property without validation.
+            // get override platform if specified
+            IGDB.Models.Platform? OverridePlatform = null;
+            if (OverridePlatformId != null)
+            {
+                OverridePlatform = Platforms.GetPlatform((long)OverridePlatformId);
+            }
 
+            // Process uploaded files
             foreach (Dictionary<string, object> UploadedFile in UploadedFiles)
             {
-                Classes.ImportGame.ImportGameFile((string)UploadedFile["fullpath"]);
+                Classes.ImportGame.ImportGameFile((string)UploadedFile["fullpath"], OverridePlatform, false);
             }
 
             if (Directory.Exists(workPath))

gaseous-server/Controllers/SignaturesController.cs

@@ -55,7 +55,7 @@ namespace gaseous_server.Controllers
         private List<Models.Signatures_Games> _GetSignature(string sqlWhere, string searchString)
         {
             Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-            string sql = "SELECT     view_Signatures_Games.*,    Signatures_Roms.Id AS romid,    Signatures_Roms.Name AS romname,    Signatures_Roms.Size,    Signatures_Roms.CRC,    Signatures_Roms.MD5,    Signatures_Roms.SHA1,    Signatures_Roms.DevelopmentStatus,    Signatures_Roms.Flags,    Signatures_Roms.RomType,    Signatures_Roms.RomTypeMedia,    Signatures_Roms.MediaLabel,    Signatures_Roms.MetadataSource FROM    Signatures_Roms        INNER JOIN    view_Signatures_Games ON Signatures_Roms.GameId = view_Signatures_Games.Id WHERE " + sqlWhere;
+            string sql = "SELECT     view_Signatures_Games.*,    Signatures_Roms.Id AS romid,    Signatures_Roms.Name AS romname,    Signatures_Roms.Size,    Signatures_Roms.CRC,    Signatures_Roms.MD5,    Signatures_Roms.SHA1,    Signatures_Roms.DevelopmentStatus,    Signatures_Roms.Attributes,    Signatures_Roms.RomType,    Signatures_Roms.RomTypeMedia,    Signatures_Roms.MediaLabel,    Signatures_Roms.MetadataSource FROM    Signatures_Roms        INNER JOIN    view_Signatures_Games ON Signatures_Roms.GameId = view_Signatures_Games.Id WHERE " + sqlWhere;
             Dictionary<string, object> dbDict = new Dictionary<string, object>();
             dbDict.Add("searchString", searchString);
 
@@ -88,14 +88,14 @@ namespace gaseous_server.Controllers
                         Name = (string)sigDbRow["romname"],
                         Size = (Int64)sigDbRow["Size"],
                         Crc = (string)sigDbRow["CRC"],
-                        Md5 = (string)sigDbRow["MD5"],
-                        Sha1 = (string)sigDbRow["SHA1"],
+                        Md5 = ((string)sigDbRow["MD5"]).ToLower(),
+                        Sha1 = ((string)sigDbRow["SHA1"]).ToLower(),
                         DevelopmentStatus = (string)sigDbRow["DevelopmentStatus"],
-                        flags = Newtonsoft.Json.JsonConvert.DeserializeObject<List<string>>((string)sigDbRow["Flags"]),
+                        Attributes = Newtonsoft.Json.JsonConvert.DeserializeObject<List<KeyValuePair<string, object>>>((string)Common.ReturnValueIfNull(sigDbRow["Attributes"], "[]")),
                         RomType = (Models.Signatures_Games.RomItem.RomTypes)(int)sigDbRow["RomType"],
                         RomTypeMedia = (string)sigDbRow["RomTypeMedia"],
                         MediaLabel = (string)sigDbRow["MediaLabel"],
-                        SignatureSource = (Models.Signatures_Games.RomItem.SignatureSourceType)(Int32)sigDbRow["MetadataSource"]
+                        SignatureSource = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(Int32)sigDbRow["MetadataSource"]
                     }
                 };
                 GamesList.Add(gameItem);

gaseous-server/Controllers/SystemController.cs

@@ -62,7 +62,9 @@ namespace gaseous_server.Controllers
         [Route("VersionFile")]
         [ProducesResponseType(StatusCodes.Status200OK)]
         public FileContentResult GetSystemVersionAsFile() {
-            string ver = "var AppVersion = \"" + Assembly.GetExecutingAssembly().GetName().Version.ToString() + "\"";
+            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string ver = "var AppVersion = \"" + Assembly.GetExecutingAssembly().GetName().Version.ToString() + "\";" + Environment.NewLine +
+                "var DBSchemaVersion = \"" + db.GetDatabaseSchemaVersion() + "\";";
             byte[] bytes = Encoding.UTF8.GetBytes(ver);
             return File(bytes, "text/javascript");
         }

gaseous-server/Models/PlatformMapping.cs

@@ -1,54 +1,348 @@
 using System;
+using System.Collections;
+using System.Data;
+using System.Linq;
 using System.Reflection;
 using System.Text.Json.Serialization;
 using gaseous_server.Classes;
+using gaseous_server.Classes.Metadata;
+using gaseous_server.Controllers;
+using gaseous_tools;
+using IGDB.Models;
+using Newtonsoft.Json;
 
 namespace gaseous_server.Models
 {
 	public class PlatformMapping
 	{
-        public PlatformMapping()
+        /// <summary>
+        /// Updates the platform map from the embedded platform map resource
+        /// </summary>
+        public static void ExtractPlatformMap(bool ResetToDefault = false)
         {
+            using (Stream stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("gaseous_server.Support.PlatformMap.json"))
+            using (StreamReader reader = new StreamReader(stream))
+            {
+                string rawJson = reader.ReadToEnd();
+                List<PlatformMapItem> platforms = new List<PlatformMapItem>();
+                Newtonsoft.Json.JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings{
+                    MaxDepth = 64
+                };
+                platforms = Newtonsoft.Json.JsonConvert.DeserializeObject<List<PlatformMapItem>>(rawJson, jsonSerializerSettings);
 
+                foreach (PlatformMapItem mapItem in platforms)
+                {
+                    // check if it exists first - only add if it doesn't exist
+                    try
+                    {
+                        Logging.Log(Logging.LogType.Information, "Platform Map", "Checking if " + mapItem.IGDBName + " is in database.");
+                        PlatformMapItem item = GetPlatformMap(mapItem.IGDBId);
+                        // exists
+                        if (ResetToDefault == false)
+                        {
+                            Logging.Log(Logging.LogType.Information, "Platform Map", "Skipping import of " + mapItem.IGDBName + " - already in database.");
+                        }
+                        else
+                        {
+                            WritePlatformMap(mapItem, true, true);
+                            Logging.Log(Logging.LogType.Information, "Platform Map", "Overwriting " + mapItem.IGDBName + " with default values.");
+                        }
+                    }
+                    catch
+                    {
+                        Logging.Log(Logging.LogType.Information, "Platform Map", "Importing " + mapItem.IGDBName + " from predefined data.");
+                        // doesn't exist - add it
+                        WritePlatformMap(mapItem, false, true);
+                    }
+                }
+            }
         }
 
-        //private static List<PlatformMapItem> _PlatformMaps = new List<PlatformMapItem>();
+        /// <summary>
+        /// Updates the platform map from the provided file - existing items are overwritten
+        /// </summary>
+        /// <param name="ImportFile"></param>
+        public static void ExtractPlatformMap(string ImportFile)
+        {
+            string rawJson = File.ReadAllText(ImportFile);
+            List<PlatformMapItem> platforms = new List<PlatformMapItem>();
+            platforms = Newtonsoft.Json.JsonConvert.DeserializeObject<List<PlatformMapItem>>(rawJson);
+
+            foreach (PlatformMapItem mapItem in platforms)
+            {
+                try
+                {
+                    PlatformMapItem item = GetPlatformMap(mapItem.IGDBId);
+
+                    // still here? we must have found the item we're looking for! overwrite it
+                    Logging.Log(Logging.LogType.Information, "Platform Map", "Replacing " + mapItem.IGDBName + " from external JSON file.");
+                    WritePlatformMap(mapItem, true, true);
+                }
+                catch
+                {
+                    // we caught a not found error, insert a new record
+                    Logging.Log(Logging.LogType.Information, "Platform Map", "Importing " + mapItem.IGDBName + " from external JSON file.");
+                    WritePlatformMap(mapItem, false, true);
+                }
+            }
+        }
+        
         public static List<PlatformMapItem> PlatformMap
         {
             get
             {
-                // load platform maps from: gaseous_server.Support.PlatformMap.json
-                List<PlatformMapItem> _PlatformMaps = new List<PlatformMapItem>();
-                var assembly = Assembly.GetExecutingAssembly();
-                var resourceName = "gaseous_server.Support.PlatformMap.json";
-                using (Stream stream = assembly.GetManifestResourceStream(resourceName))
-                using (StreamReader reader = new StreamReader(stream))
+                Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+                string sql = "SELECT * FROM PlatformMap";
+                DataTable data = db.ExecuteCMD(sql);
+
+                List<PlatformMapItem> platformMaps = new List<PlatformMapItem>();
+                foreach (DataRow row in data.Rows)
                 {
-                    string rawJson = reader.ReadToEnd();
-                    _PlatformMaps.Clear();
-                    _PlatformMaps = Newtonsoft.Json.JsonConvert.DeserializeObject<List<PlatformMapItem>>(rawJson);
+                    platformMaps.Add(BuildPlatformMapItem(row));
                 }
 
-                return _PlatformMaps;
+                platformMaps.Sort((x, y) => x.IGDBName.CompareTo(y.IGDBName));
+
+                return platformMaps;
             }
         }
 
+        public static PlatformMapItem GetPlatformMap(long Id)
+        {
+            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "SELECT * FROM PlatformMap WHERE Id = @Id";
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+            dbDict.Add("Id", Id);
+            DataTable data = db.ExecuteCMD(sql, dbDict);
+
+            if (data.Rows.Count > 0)
+            {
+                PlatformMapItem platformMap = BuildPlatformMapItem(data.Rows[0]);
+
+                return platformMap;
+            }
+            else
+            {
+                Exception exception = new Exception("Platform Map Id " + Id + " does not exist.");
+                Logging.Log(Logging.LogType.Critical, "Platform Map", "Platform Map Id " + Id + " does not exist.", exception);
+                throw exception;
+            }
+        }
+
+        public static void WritePlatformMap(PlatformMapItem item, bool Update, bool AllowAvailableEmulatorOverwrite)
+        {
+            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "";
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+            if (Update == false)
+            {
+                // insert
+                sql = "INSERT INTO PlatformMap (Id, RetroPieDirectoryName, WebEmulator_Type, WebEmulator_Core, AvailableWebEmulators) VALUES (@Id, @RetroPieDirectoryName, @WebEmulator_Type, @WebEmulator_Core, @AvailableWebEmulators)";
+            }
+            else
+            {
+                // update
+                if (AllowAvailableEmulatorOverwrite == true)
+                {
+                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core, AvailableWebEmulators=@AvailableWebEmulators WHERE Id = @Id";
+                }
+                else
+                {
+                    sql = "UPDATE PlatformMap SET RetroPieDirectoryName=@RetroPieDirectoryName, WebEmulator_Type=@WebEmulator_Type, WebEmulator_Core=@WebEmulator_Core WHERE Id = @Id";
+                }
+            }
+            dbDict.Add("Id", item.IGDBId);
+            dbDict.Add("RetroPieDirectoryName", item.RetroPieDirectoryName);
+            if (item.WebEmulator != null)
+            {
+                dbDict.Add("WebEmulator_Type", item.WebEmulator.Type);
+                dbDict.Add("WebEmulator_Core", item.WebEmulator.Core);
+                dbDict.Add("AvailableWebEmulators", Newtonsoft.Json.JsonConvert.SerializeObject(item.WebEmulator.AvailableWebEmulators));
+            }
+            else
+            {
+                dbDict.Add("WebEmulator_Type", "");
+                dbDict.Add("WebEmulator_Core", "");
+                dbDict.Add("AvailableWebEmulators", "");
+            }
+            db.ExecuteCMD(sql, dbDict);
+
+            // remove existing items so they can be re-inserted
+            sql = "DELETE FROM PlatformMap_AlternateNames WHERE Id = @Id; DELETE FROM PlatformMap_Extensions WHERE Id = @Id; DELETE FROM PlatformMap_UniqueExtensions WHERE Id = @Id; DELETE FROM PlatformMap_Bios WHERE Id = @Id;";
+            db.ExecuteCMD(sql, dbDict);
+
+            // insert alternate names
+            if (item.AlternateNames != null)
+            {
+                foreach (string alternateName in item.AlternateNames)
+                {
+                    if (alternateName != null)
+                    {
+                        sql = "INSERT INTO PlatformMap_AlternateNames (Id, Name) VALUES (@Id, @Name);";
+                        dbDict.Clear();
+                        dbDict.Add("Id", item.IGDBId);
+                        dbDict.Add("Name", alternateName);
+                        db.ExecuteCMD(sql, dbDict);
+                    }
+                }
+            }
+
+            // insert extensions
+            if (item.Extensions != null)
+            {
+                foreach (string extension in item.Extensions.SupportedFileExtensions)
+                {
+                    sql = "INSERT INTO PlatformMap_Extensions (Id, Extension) VALUES (@Id, @Extension);";
+                    dbDict.Clear();
+                    dbDict.Add("Id", item.IGDBId);
+                    dbDict.Add("Extension", extension.Trim().ToUpper());
+                    db.ExecuteCMD(sql, dbDict);
+                }
+
+                // delete duplicates
+                sql = "DELETE FROM PlatformMap_UniqueExtensions; INSERT INTO PlatformMap_UniqueExtensions SELECT * FROM PlatformMap_Extensions WHERE Extension <> '.ZIP' AND Extension IN (SELECT Extension FROM PlatformMap_Extensions GROUP BY Extension HAVING COUNT(Extension) = 1);";
+                db.ExecuteCMD(sql);
+            }
+
+            // insert bios
+            if (item.Bios != null)
+            {
+                foreach (PlatformMapItem.EmulatorBiosItem biosItem in item.Bios)
+                {
+                    sql = "INSERT INTO PlatformMap_Bios (Id, Filename, Description, Hash) VALUES (@Id, @Filename, @Description, @Hash);";
+                    dbDict.Clear();
+                    dbDict.Add("Id", item.IGDBId);
+                    dbDict.Add("Filename", biosItem.filename);
+                    dbDict.Add("Description", biosItem.description);
+                    dbDict.Add("Hash", biosItem.hash);
+                    db.ExecuteCMD(sql, dbDict);
+                }
+            }
+        }
+
+        static PlatformMapItem BuildPlatformMapItem(DataRow row)
+        {
+            long IGDBId = (long)row["Id"];
+            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+            string sql = "";
+
+            // get platform data
+            IGDB.Models.Platform platform = Platforms.GetPlatform(IGDBId);
+
+            // get platform alternate names
+            sql = "SELECT * FROM PlatformMap_AlternateNames WHERE Id = @Id ORDER BY Name";
+            dbDict.Clear();
+            dbDict.Add("Id", IGDBId);
+            DataTable altTable = db.ExecuteCMD(sql, dbDict);
+
+            List<string> alternateNames = new List<string>();
+            foreach (DataRow altRow in altTable.Rows)
+            {
+                string altVal = (string)altRow["Name"];
+                if (!alternateNames.Contains(altVal, StringComparer.OrdinalIgnoreCase))
+                {
+                    alternateNames.Add(altVal);
+                }
+            }
+            if (platform.AlternativeName != null)
+            {
+                if (!alternateNames.Contains(platform.AlternativeName, StringComparer.OrdinalIgnoreCase))
+                {
+                    alternateNames.Add(platform.AlternativeName);
+                }
+            }
+
+            // get platform known extensions
+            sql = "SELECT * FROM PlatformMap_Extensions WHERE Id = @Id ORDER BY Extension";
+            dbDict.Clear();
+            dbDict.Add("Id", IGDBId);
+            DataTable extTable = db.ExecuteCMD(sql, dbDict);
+
+            List<string> knownExtensions = new List<string>();
+            foreach (DataRow extRow in extTable.Rows)
+            {
+                string extVal = (string)extRow["Extension"];
+                if (!knownExtensions.Contains(extVal, StringComparer.OrdinalIgnoreCase))
+                {
+                    knownExtensions.Add(extVal);
+                }
+            }
+
+            // get platform unique extensions
+            sql = "SELECT * FROM PlatformMap_UniqueExtensions WHERE Id = @Id ORDER BY Extension";
+            dbDict.Clear();
+            dbDict.Add("Id", IGDBId);
+            DataTable uextTable = db.ExecuteCMD(sql, dbDict);
+
+            List<string> uniqueExtensions = new List<string>();
+            foreach (DataRow uextRow in uextTable.Rows)
+            {
+                string uextVal = (string)uextRow["Extension"];
+                if (!uniqueExtensions.Contains(uextVal, StringComparer.OrdinalIgnoreCase))
+                {
+                    uniqueExtensions.Add(uextVal);
+                }
+            }
+
+            // get platform bios
+            sql = "SELECT * FROM PlatformMap_Bios WHERE Id = @Id ORDER BY Filename";
+            dbDict.Clear();
+            dbDict.Add("Id", IGDBId);
+            DataTable biosTable = db.ExecuteCMD(sql, dbDict);
+
+            List<PlatformMapItem.EmulatorBiosItem> bioss = new List<PlatformMapItem.EmulatorBiosItem>();
+            foreach (DataRow biosRow in biosTable.Rows)
+            {
+                PlatformMapItem.EmulatorBiosItem bios = new PlatformMapItem.EmulatorBiosItem
+                {
+                    filename = (string)Common.ReturnValueIfNull(biosRow["Filename"], ""),
+                    description = (string)Common.ReturnValueIfNull(biosRow["Description"], ""),
+                    hash = ((string)Common.ReturnValueIfNull(biosRow["Hash"], "")).ToLower()
+                };
+                bioss.Add(bios);
+            }
+
+            // build item
+            PlatformMapItem mapItem = new PlatformMapItem();
+            mapItem.IGDBId = IGDBId;
+            mapItem.IGDBName = platform.Name;
+            mapItem.IGDBSlug = platform.Slug;
+            mapItem.AlternateNames = alternateNames;
+            mapItem.Extensions = new PlatformMapItem.FileExtensions{
+                SupportedFileExtensions = knownExtensions,
+                UniqueFileExtensions = uniqueExtensions
+            };
+            mapItem.RetroPieDirectoryName = (string)Common.ReturnValueIfNull(row["RetroPieDirectoryName"], "");
+            mapItem.WebEmulator = new PlatformMapItem.WebEmulatorItem{
+                Type = (string)Common.ReturnValueIfNull(row["WebEmulator_Type"], ""),
+                Core = (string)Common.ReturnValueIfNull(row["WebEmulator_Core"], ""),
+                AvailableWebEmulators = Newtonsoft.Json.JsonConvert.DeserializeObject<List<PlatformMapItem.WebEmulatorItem.AvailableWebEmulatorItem>>((string)Common.ReturnValueIfNull(row["AvailableWebEmulators"], "[]"))
+            };
+            mapItem.Bios = bioss;
+            
+            return mapItem;
+        }
+
         public static void GetIGDBPlatformMapping(ref Models.Signatures_Games Signature, FileInfo RomFileInfo, bool SetSystemName)
         {
             bool PlatformFound = false;
             foreach (Models.PlatformMapping.PlatformMapItem PlatformMapping in Models.PlatformMapping.PlatformMap)
             {
-                if (PlatformMapping.KnownFileExtensions.Contains(RomFileInfo.Extension, StringComparer.OrdinalIgnoreCase))
+                if (PlatformMapping.Extensions != null)
                 {
-                    if (SetSystemName == true)
+                    if (PlatformMapping.Extensions.UniqueFileExtensions.Contains(RomFileInfo.Extension, StringComparer.OrdinalIgnoreCase))
                     {
-                        if (Signature.Game != null) { Signature.Game.System = PlatformMapping.IGDBName; }
-                    }
-                    Signature.Flags.IGDBPlatformId = PlatformMapping.IGDBId;
-                    Signature.Flags.IGDBPlatformName = PlatformMapping.IGDBName;
+                        if (SetSystemName == true)
+                        {
+                            if (Signature.Game != null) { Signature.Game.System = PlatformMapping.IGDBName; }
+                        }
+                        Signature.Flags.IGDBPlatformId = PlatformMapping.IGDBId;
+                        Signature.Flags.IGDBPlatformName = PlatformMapping.IGDBName;
 
-                    PlatformFound = true;
-                    break;
+                        PlatformFound = true;
+                        break;
+                    }
                 }
             }
 
@@ -56,7 +350,10 @@ namespace gaseous_server.Models
             {
                 foreach (Models.PlatformMapping.PlatformMapItem PlatformMapping in Models.PlatformMapping.PlatformMap)
                 {
-                    if (PlatformMapping.AlternateNames.Contains(Signature.Game.System, StringComparer.OrdinalIgnoreCase))
+                    if (
+                        PlatformMapping.IGDBName == Signature.Game.System ||
+                        PlatformMapping.AlternateNames.Contains(Signature.Game.System, StringComparer.OrdinalIgnoreCase)
+                        )
                     {
                         if (SetSystemName == true)
                         {
@@ -74,27 +371,51 @@ namespace gaseous_server.Models
 
         public class PlatformMapItem
         {
-            public int IGDBId { get; set; }
+            public long IGDBId { get; set; }
             public string IGDBName { get; set; }
+            public string IGDBSlug { get; set; }
             public List<string> AlternateNames { get; set; } = new List<string>();
-            public List<string> KnownFileExtensions { get; set; } = new List<string>();
-            //public Dictionary<string, object>? WebEmulator { get; set; }
+            
+            public FileExtensions Extensions { get; set; }
+            public class FileExtensions
+            {
+                public List<string> SupportedFileExtensions { get; set; } = new List<string>();
+
+                public List<string> UniqueFileExtensions { get; set; } = new List<string>();
+            }
+
+            public string RetroPieDirectoryName { get; set; }
             public WebEmulatorItem? WebEmulator { get; set; }
 
             public class WebEmulatorItem
             {
                 public string Type { get; set; }
                 public string Core { get; set; }
-                public List<EmulatorBiosItem> Bios { get; set; }
 
-                public class EmulatorBiosItem
+                public List<AvailableWebEmulatorItem> AvailableWebEmulators { get; set; } = new List<AvailableWebEmulatorItem>();
+
+                public class AvailableWebEmulatorItem
                 {
-                    public string hash { get; set; }
-                    public string description { get; set; }
-                    public string filename { get; set; }
-                    public string region { get; set; }
+                    public string EmulatorType { get; set; }
+                    public List<AvailableWebEmulatorCoreItem> AvailableWebEmulatorCores { get; set; } = new List<AvailableWebEmulatorCoreItem>();
+
+                    public class AvailableWebEmulatorCoreItem
+                    {
+                        public string Core { get; set; }
+                        public string? AlternateCoreName { get; set; } = "";
+                        public bool Default { get; set; } = false;
+                    }
                 }
             }
+
+            public List<EmulatorBiosItem> Bios { get; set; }
+
+            public class EmulatorBiosItem
+            {
+                public string hash { get; set; }
+                public string description { get; set; }
+                public string filename { get; set; }
+            }
         }
 	}
 }

gaseous-server/Models/Signatures_Games.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Text.Json.Serialization;
-using static gaseous_romsignatureobject.RomSignatureObject.Game;
 
 namespace gaseous_server.Models
 {
@@ -127,19 +126,13 @@ namespace gaseous_server.Models
 
             public string? DevelopmentStatus { get; set; }
 
-            public List<string> flags { get; set; } = new List<string>();
+            public List<KeyValuePair<string, object>> Attributes { get; set; } = new List<KeyValuePair<string, object>>();
 
             public RomTypes RomType { get; set; }
             public string? RomTypeMedia { get; set; }
             public string? MediaLabel { get; set; }
 
-            public SignatureSourceType SignatureSource { get; set; }
-
-            public enum SignatureSourceType
-            {
-                None = 0,
-                TOSEC = 1
-            }
+            public gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType SignatureSource { get; set; }
 
             public enum RomTypes
             {
@@ -198,6 +191,7 @@ namespace gaseous_server.Models
                                 case "crc":
                                 case "developmentstatus":
                                 case "flags":
+                                case "attributes":
                                 case "romtypemedia":
                                 case "medialabel":
                                     if (prop.PropertyType == typeof(string) || prop.PropertyType == typeof(Int64) || prop.PropertyType == typeof(List<string>))

gaseous-server/ProcessQueue.cs

@@ -9,34 +9,52 @@ namespace gaseous_server
 
         public class QueueItem
         {
-            public QueueItem(QueueItemType ItemType, int ExecutionInterval, bool AllowManualStart = true)
+            public QueueItem(QueueItemType ItemType, int ExecutionInterval, bool AllowManualStart = true, bool RemoveWhenStopped = false)
             {
                 _ItemType = ItemType;
                 _ItemState = QueueItemState.NeverStarted;
-                _LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval);
+                _LastRunTime = DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
                 _Interval = ExecutionInterval;
                 _AllowManualStart = AllowManualStart;
+                _RemoveWhenStopped = RemoveWhenStopped;
             }
 
-            public QueueItem(QueueItemType ItemType, int ExecutionInterval, List<QueueItemType> Blocks, bool AllowManualStart = true)
+            public QueueItem(QueueItemType ItemType, int ExecutionInterval, List<QueueItemType> Blocks, bool AllowManualStart = true, bool RemoveWhenStopped = false)
             {
                 _ItemType = ItemType;
                 _ItemState = QueueItemState.NeverStarted;
-                _LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval);
+                _LastRunTime = DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
                 _Interval = ExecutionInterval;
                 _AllowManualStart = AllowManualStart;
+                _RemoveWhenStopped = RemoveWhenStopped;
                 _Blocks = Blocks;
             }
 
             private QueueItemType _ItemType = QueueItemType.NotConfigured;
             private QueueItemState _ItemState = QueueItemState.NeverStarted;
             private DateTime _LastRunTime = DateTime.UtcNow;
-            private DateTime _LastFinishTime = DateTime.UtcNow;
+            private DateTime _LastFinishTime
+            {
+                get
+                {
+                    return DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
+                }
+                set
+                {
+                    if (_SaveLastRunTime == true)
+                    {
+                        Config.SetSetting("LastRun_" + _ItemType.ToString(), value.ToString("yyyy-MM-ddThh:mm:ssZ"));
+                    }
+                }
+            }
+            private bool _SaveLastRunTime = false;
             private int _Interval = 0;
             private string _LastResult = "";
             private string? _LastError = null;
             private bool _ForceExecute = false;
             private bool _AllowManualStart = true;
+            private bool _RemoveWhenStopped = false;
+            private bool _IsBlocked = false;
             private List<QueueItemType> _Blocks = new List<QueueItemType>();
 
             public QueueItemType ItemType => _ItemType;
@@ -54,6 +72,9 @@ namespace gaseous_server
             public string? LastError => _LastError;
             public bool Force => _ForceExecute;
             public bool AllowManualStart => _AllowManualStart;
+            public bool RemoveWhenStopped => _RemoveWhenStopped;
+            public bool IsBlocked => _IsBlocked;
+            public object? Options { get; set; } = null;
             public List<QueueItemType> Blocks => _Blocks;
 
             public void Execute()
@@ -68,42 +89,71 @@ namespace gaseous_server
                         _LastResult = "";
                         _LastError = null;
 
-                        Logging.Log(Logging.LogType.Information, "Timered Event", "Executing " + _ItemType);
+                        Logging.Log(Logging.LogType.Debug, "Timered Event", "Executing " + _ItemType);
 
                         try
                         {
                             switch (_ItemType)
                             {
                                 case QueueItemType.SignatureIngestor:
-                                    Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Signature Ingestor");
-                                    SignatureIngestors.TOSEC.TOSECIngestor tIngest = new SignatureIngestors.TOSEC.TOSECIngestor();
-                                    tIngest.Import(Config.LibraryConfiguration.LibrarySignatureImportDirectory_TOSEC);
+                                    Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Signature Ingestor");
+                                    SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor();
+                                    
+                                    Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing TOSEC files");
+                                    tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "TOSEC"), gaseous_signature_parser.parser.SignatureParser.TOSEC);
+                                    
+                                    Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME Arcade files");
+                                    tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME Arcade"), gaseous_signature_parser.parser.SignatureParser.MAMEArcade);
+
+                                    Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME MESS files");
+                                    tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess);
+                                    
+                                    _SaveLastRunTime = true;
+
                                     break;
 
                                 case QueueItemType.TitleIngestor:
-                                    Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Title Ingestor");
+                                    Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor");
                                     Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory);
+
+                                    _SaveLastRunTime = true;
+
                                     break;
 
                                 case QueueItemType.MetadataRefresh:
-                                    Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Metadata Refresher");
+                                    Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher");
                                     Classes.MetadataManagement.RefreshMetadata(true);
+
+                                    _SaveLastRunTime = true;
+
                                     break;
 
                                 case QueueItemType.OrganiseLibrary:
-                                    Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Library Organiser");
+                                    Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Organiser");
                                     Classes.ImportGame.OrganiseLibrary();
+
+                                    _SaveLastRunTime = true;
+
                                     break;
 
                                 case QueueItemType.LibraryScan:
-                                    Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Library Scanner");
+                                    Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner");
                                     Classes.ImportGame.LibraryScan();
+
+                                    _SaveLastRunTime = true;
+
                                     break;
 
                                 case QueueItemType.CollectionCompiler:
-                                    Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Collection Compiler");
-                                    Classes.Collections.CompileCollections();
+                                    Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
+                                    Classes.Collections.CompileCollections((long)Options);
                                     break;
+
+                                case QueueItemType.BackgroundDatabaseUpgrade:
+                                    Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Background Upgrade");
+                                    gaseous_tools.DatabaseMigration.UpgradeScriptBackgroundTasks();
+                                    break;
+
                             }
                         }
                         catch (Exception ex)
@@ -124,17 +174,59 @@ namespace gaseous_server
             {
                 _ForceExecute = true;
             }
+
+            public void BlockedState(bool BlockState)
+            {
+                _IsBlocked = BlockState;
+            }
         }
 
         public enum QueueItemType
         {
+            /// <summary>
+            /// Reserved for blocking all services - no actual background service is tied to this type
+            /// </summary>
+            All,
+
+            /// <summary>
+            /// Default type - no background service is tied to this type
+            /// </summary>
             NotConfigured,
+
+            /// <summary>
+            /// Ingests signature DAT files into the database
+            /// </summary>
             SignatureIngestor,
+
+            /// <summary>
+            /// Imports game files into the database and moves them to the required location on disk
+            /// </summary>
             TitleIngestor,
+
+            /// <summary>
+            /// Forces stored metadata to be refreshed
+            /// </summary>
             MetadataRefresh,
+
+            /// <summary>
+            /// Ensures all managed files are where they are supposed to be
+            /// </summary>
             OrganiseLibrary,
+
+            /// <summary>
+            /// Looks for orphaned files in the library and re-adds them to the database
+            /// </summary>
             LibraryScan,
-            CollectionCompiler
+
+            /// <summary>
+            /// Builds collections - set the options attribute to the id of the collection to build
+            /// </summary>
+            CollectionCompiler,
+
+            /// <summary>
+            /// Performs and post database upgrade scripts that can be processed as a background task
+            /// </summary>
+            BackgroundDatabaseUpgrade
         }
 
         public enum QueueItemState

gaseous-server/Program.cs

@@ -1,12 +1,15 @@
 using System.Reflection;
 using System.Text.Json.Serialization;
 using gaseous_server;
+using gaseous_server.Models;
+using gaseous_server.SignatureIngestors.XML;
 using gaseous_tools;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.OpenApi.Models;
 
+Logging.WriteToDiskOnly = true;
 Logging.Log(Logging.LogType.Information, "Startup", "Starting Gaseous Server " + Assembly.GetExecutingAssembly().GetName().Version);
 
 // set up db
@@ -15,6 +18,8 @@ db.InitDB();
 
 // load app settings
 Config.InitSettings();
+// write updated settings back to the config file
+Config.UpdateConfig();
 
 // set initial values
 Guid APIKey = Guid.NewGuid();
@@ -24,11 +29,23 @@ if (Config.ReadSetting("API Key", "Test API Key") == "Test API Key")
     Logging.Log(Logging.LogType.Information, "Startup", "Setting initial API key");
     Config.SetSetting("API Key", APIKey.ToString());
 }
-if (Config.ReadSetting("Emulator: Default BIOS Region", "Default Value") == "Default Value")
-{
-    Logging.Log(Logging.LogType.Information, "Startup", "Setting default BIOS region to US");
-    Config.SetSetting("Emulator: Default BIOS Region", "US");
-}
+
+// kick off any delayed upgrade tasks
+// run 1002 background updates in the background on every start
+DatabaseMigration.BackgroundUpgradeTargetSchemaVersions.Add(1002);
+// start the task
+ProcessQueue.QueueItem queueItem = new ProcessQueue.QueueItem(
+        ProcessQueue.QueueItemType.BackgroundDatabaseUpgrade,
+        1,
+        new List<ProcessQueue.QueueItemType>
+        {
+            ProcessQueue.QueueItemType.All
+        },
+        false,
+        true
+    );
+queueItem.ForceExecute();
+ProcessQueue.QueueItems.Add(queueItem);
 
 // set up server
 var builder = WebApplication.CreateBuilder(args);
@@ -143,8 +160,8 @@ Config.LibraryConfiguration.InitLibrary();
 gaseous_server.Classes.Metadata.Games.GetGame(0, false, false, false);
 gaseous_server.Classes.Metadata.Platforms.GetPlatform(0);
 
-// organise library
-//gaseous_server.Classes.ImportGame.OrganiseLibrary();
+// extract platform map if not present
+PlatformMapping.ExtractPlatformMap();
 
 // add background tasks
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60));
@@ -158,20 +175,20 @@ ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
     );
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.MetadataRefresh, 360));
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
-    ProcessQueue.QueueItemType.OrganiseLibrary, 2040, new List<ProcessQueue.QueueItemType>
+    ProcessQueue.QueueItemType.OrganiseLibrary, 1440, new List<ProcessQueue.QueueItemType>
     {
         ProcessQueue.QueueItemType.LibraryScan,
         ProcessQueue.QueueItemType.TitleIngestor
     })
     );
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
-    ProcessQueue.QueueItemType.LibraryScan, 30, new List<ProcessQueue.QueueItemType>
+    ProcessQueue.QueueItemType.LibraryScan, 1440, new List<ProcessQueue.QueueItemType>
     {
-        ProcessQueue.QueueItemType.TitleIngestor,
         ProcessQueue.QueueItemType.OrganiseLibrary
     })
     );
-ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.CollectionCompiler, 5, false));
+
+Logging.WriteToDiskOnly = false;
 
 // start the app
 app.Run();

gaseous-server/Timer.cs

@@ -33,9 +33,23 @@ namespace gaseous_server
             //_logger.LogInformation(
             //    "Timed Hosted Service is working. Count: {Count}", count);
 
-            foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) {
-                if ((DateTime.UtcNow > qi.NextRunTime || qi.Force == true) && CheckProcessBlockList(qi) == true) {
-                    qi.Execute();
+            List<ProcessQueue.QueueItem> ActiveList = new List<ProcessQueue.QueueItem>();
+            ActiveList.AddRange(ProcessQueue.QueueItems);
+            foreach (ProcessQueue.QueueItem qi in ActiveList) {
+                if (CheckIfProcessIsBlockedByOthers(qi) == false) {
+                    qi.BlockedState(false);
+                    if (DateTime.UtcNow > qi.NextRunTime || qi.Force == true)
+                    {
+                        qi.Execute();
+                        if (qi.RemoveWhenStopped == true && qi.ItemState == ProcessQueue.QueueItemState.Stopped)
+                        {
+                            ProcessQueue.QueueItems.Remove(qi);
+                        }
+                    }
+                }
+                else
+                {
+                    qi.BlockedState(true);
                 }
             }
         }
@@ -55,24 +69,24 @@ namespace gaseous_server
             _timer?.Dispose();
         }
 
-        private bool CheckProcessBlockList(ProcessQueue.QueueItem queueItem)
+        private bool CheckIfProcessIsBlockedByOthers(ProcessQueue.QueueItem queueItem)
         {
-            if (queueItem.Blocks.Count > 0)
+            foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
             {
-                foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
-                {
-                    if (queueItem.Blocks.Contains(qi.ItemType) && qi.ItemState == ProcessQueue.QueueItemState.Running)
+                if (qi.ItemState == ProcessQueue.QueueItemState.Running) {
+                    // other service is running, check if queueItem is blocked by it
+                    if (
+                        qi.Blocks.Contains(queueItem.ItemType) ||
+                        qi.Blocks.Contains(ProcessQueue.QueueItemType.All)
+                    )
                     {
-                        return false;
+                        Console.WriteLine(queueItem.ItemType.ToString() + " is blocked by " + qi.ItemType.ToString());
+                        return true;
                     }
                 }
+            }
 
-                return true;
-            }
-            else
-            {
-                return true;
-            }
+            return false;
         }
     }
 }

gaseous-server/gaseous-server.csproj

@@ -18,10 +18,11 @@
     <DocumentationFile>bin\Release\net7.0\gaseous-server.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.10" />
+    <PackageReference Include="gaseous-signature-parser" Version="2.0.0" />
+    <PackageReference Include="gaseous.IGDB" Version="1.0.1" />
+    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.11" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="7.0.9" />
-    <PackageReference Include="IGDB" Version="2.3.2" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="7.0.10" />
   </ItemGroup>
 
   <ItemGroup>
@@ -112,12 +113,6 @@
     <ProjectReference Include="..\gaseous-tools\gaseous-tools.csproj">
       <GlobalPropertiesToRemove></GlobalPropertiesToRemove>
     </ProjectReference>
-    <ProjectReference Include="..\gaseous-romsignatureobject\gaseous-romsignatureobject.csproj">
-      <GlobalPropertiesToRemove></GlobalPropertiesToRemove>
-    </ProjectReference>
-    <ProjectReference Include="..\gaseous-signature-parser\gaseous-signature-parser.csproj">
-      <GlobalPropertiesToRemove></GlobalPropertiesToRemove>
-    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
     <EmbeddedResource Include="Support\PlatformMap.json" Condition="'$(ExcludeConfigFilesFromBuildOutput)'!='true'">

gaseous-server/wwwroot/images/help.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M12 17H12.01M12 14C12.8906 12.0938 15 12.2344 15 10C15 8.5 14 7 12 7C10.4521 7 9.50325 7.89844 9.15332 9M12 21C16.9706 21 21 16.9706 21 12C21 7.02944 16.9706 3 12 3C7.02944 3 3 7.02944 3 12C3 16.9706 7.02944 21 12 21Z" stroke="#000000" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

gaseous-server/wwwroot/images/map.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M10 7L8.364 1H2.636L1 7zM7.6 2l1.09 4H2.31L3.4 2zm8.036 15l-.818 3H6v-7h8.545L14 15h9l-1.636-6h-5.728l-.818 3H6V9h4l-.273-1H1.273L1 9h4v12h9.545L14 23h9l-1.636-6zm.764-7h4.2l1.09 4h-6.38zm-1.09 12l1.09-4h4.2l1.09 4z"/><path fill="none" d="M0 0h24v24H0z"/></svg>

gaseous-server/wwwroot/index.html

@@ -12,6 +12,7 @@
     <script src="/scripts/jquery.lazy.plugins.min.js"></script>
     <script src="/scripts/select2.min.js"></script>
     <script src="/scripts/dropzone.min.js"></script>
+    <script src="/scripts/simpleUpload.min.js"></script>
     <script src="/scripts/main.js" type="text/javascript"></script>
     <script src="/scripts/filterformating.js" type="text/javascript"></script>
     <script src="/scripts/gamesformating.js" type="text/javascript"></script>

gaseous-server/wwwroot/pages/EmulatorJS.html

@@ -26,5 +26,7 @@
 
     EJS_backgroundImage = emuBackground;
     EJS_backgroundBlur = true;
+
+    EJS_gameName = emuGameTitle;
 </script>
 <script src='/emulators/EmulatorJS/data/loader.js'></script>

gaseous-server/wwwroot/pages/dialogs/collectionaddgame.html

@@ -0,0 +1,74 @@
+<p>
+    Select collection to add game to:
+</p>
+
+<table style="width: 100%;">
+    <tr>
+        <td colspan="2">
+            <select id="collection_addgame" style="width: 100%;"></select>
+        </td>
+    </tr>
+    <tr>
+        <td style="padding-top: 15px;">
+            <input type="checkbox" id="collection_rebuild" style="margin-right: 5px;" /><label for="collection_rebuild">Rebuild Collection</label>
+        </td>
+        <td style="text-align: right; padding-top: 15px;">
+            <button id="collection_cancelbtn" value="Cancel" onclick="closeSubDialog();">Cancel</button>
+            <button id="collection_addbtn" value="Add" onclick="AddToCollection();">Add</button>
+        </td>
+    </tr>
+</table>
+
+<script type="text/javascript">
+    document.getElementById('collection_addgame').innerHTML = "<option value='0' selected='selected'>Select collection</option>";
+
+    $('#collection_addgame').select2({
+        ajax: {
+            url: '/api/v1/Collections',
+            placeholder: 'Select collection',
+            processResults: function (data) {
+                var arr = [];
+                
+                for (var i = 0; i < data.length; i++) {
+                    arr.push({
+                        id: data[i].id,
+                        text: data[i].name
+                    });
+                }
+
+                return {
+                    results: arr
+                };
+            }
+        }
+    });
+
+    function AddToCollection() {
+        var CollectionId = Number(document.getElementById('collection_addgame').value);
+        var PlatformId = modalVariables;
+        var GameId = getQueryString('id', 'int');
+        var RebuildCollection = '';
+        if (document.getElementById('collection_rebuild').checked == true) {
+            RebuildCollection = '?Rebuild=true';
+        }
+        var responseBody = {
+            "PlatformId": PlatformId,
+            "GameId": GameId,
+            "InclusionState": "AlwaysInclude"
+        };
+
+        if (CollectionId != 0) {
+            ajaxCall(
+                '/api/v1/Collections/' + CollectionId + '/AlwaysInclude' + RebuildCollection,
+                'PATCH',
+                function (result) {
+                    closeSubDialog();
+                },
+                function (error) {
+                    console.log(JSON.stringify(error));
+                },
+                JSON.stringify(responseBody)
+            );
+        }
+    }
+</script>

gaseous-server/wwwroot/pages/dialogs/collectionedit.html

@@ -56,6 +56,47 @@
                     <th>Maximum collection size (bytes)</th>
                     <td><input id="collection_maxcollectionsize" type="number" placeholder="0" step="1048576" oninput="DisplayFormattedBytes('collection_maxcollectionsize', 'maxcollectionsize_label');"><span id="maxcollectionsize_label" style="margin-left: 10px;"></span></td></td>
                 </tr>
+                <tr>
+                    <th>Directory Layout</th>
+                    <td>
+                        <select id="collection_directorylayout" style="width: 100%;" onchange="DisplayDirectoryLabel();">
+                            <option id="collection_directorylayout_gaseous" selected="selected" value="Gaseous">Standard</option>
+                            <option id="collection_directorylayout_retropie" value="RetroPie">RetroPie</option>
+                        </select>
+                    </td>
+                </tr>
+                <tr>
+                    <td>
+
+                    </td>
+                    <td>
+                        <span id="collection_directorylayout_gaseous_label" name="collection_directorylayout_label">
+                            <p>Standard layout: /&lt;IGDB Platform Slug&gt;/&lt;IGDB Game Slug&gt;/Game ROM's</p>
+                            <p>Example: /genesis-slash-megadrive/sonic-the-hedgehog/Sonic the Hedgehog.smd</p>
+                        </span>
+                        <span id="collection_directorylayout_retropie_label" style="display: none;" name="collection_directorylayout_label">
+                            <p>RetroPie layout: /roms/&lt;RetroPie Platform Label&gt;/Game ROM's</p>
+                            <p>Example: /roms/megadrive/Sonic the Hedgehog.smd</p>
+                        </span>
+                    </td>
+                </tr>
+                <tr>
+                    <th>
+                        Include BIOS files (if available)
+                    </th>
+                    <td>
+                        <select id="collection_includebios" style="width: 100%;">
+                            <option id="collection_includebios_yes" selected="selected" value="true">Yes</option>
+                            <option id="collection_includebios_no" value="false">No</option>
+                        </select>
+                    </td>
+                </tr>
+                <tr id="collection_includebios_label">
+                    <td></td>
+                    <td>
+                        BIOS files for each platform will be stored in /BIOS
+                    </td>
+                </tr>
             </table>
         </div>
         <table style="position: absolute; top: 0px; right: 0px; bottom: 0px; width: 60%;">
@@ -208,7 +249,13 @@
         }
     });
 
+    $('#collection_directorylayout').select2();
+
+    $('#collection_includebios').select2();
+
     if (modalVariables) {
+        var modalAlwaysInclude = [];
+
         // edit mode
         ajaxCall(
             '/api/v1/Collections/' + modalVariables,
@@ -221,6 +268,12 @@
                 if (result.maximumRomsPerPlatform != -1) { document.getElementById('collection_maxroms').value = result.maximumRomsPerPlatform; }
                 if (result.maximumBytesPerPlatform != -1) { document.getElementById('collection_maxplatformsize').value = result.maximumBytesPerPlatform; }
                 if (result.maximumCollectionSizeInBytes != -1) { document.getElementById('collection_maxcollectionsize').value = result.maximumCollectionSizeInBytes; }
+                if (result.folderStructure) {
+                    $('#collection_directorylayout').val(result.folderStructure).trigger('change');
+                }
+                $('#collection_includebios').val(result.includeBIOSFiles.toString()).trigger('change');
+                modalAlwaysInclude = result.alwaysInclude;
+                console.log(JSON.stringify(modalAlwaysInclude));
 
                 // fill select2 controls
                 $.ajax(
@@ -323,6 +376,28 @@
         var players = GetDropDownIds('#collection_players');
         var playerperspectives = GetDropDownIds('#collection_playerperspectives');
         var themes = GetDropDownIds('#collection_themes');
+        var alwaysInclude = [];
+        var alwaysIncludeSelections = document.getElementsByName('collections_preview_always');
+        if (alwaysIncludeSelections.length > 0) {
+            for (var i = 0; i < alwaysIncludeSelections.length; i++) {
+                if (alwaysIncludeSelections[i].value != "None") {
+                    var alwaysIncludeItem = {
+                        "platformId": Number(alwaysIncludeSelections[i].getAttribute('data-platform')),
+                        "gameId": Number(alwaysIncludeSelections[i].getAttribute('data-game')),
+                        "inclusionState": alwaysIncludeSelections[i].value
+                    };
+                    alwaysInclude.push(alwaysIncludeItem);
+                }
+            }
+        } else {
+            alwaysInclude = modalAlwaysInclude;
+        }
+
+        modalAlwaysInclude = alwaysInclude;
+
+        if (!alwaysInclude) {
+            alwaysInclude = [];
+        }
 
         var item = {
             "name": document.getElementById('collection_name').value,
@@ -336,17 +411,20 @@
             "maximumRating": GetNumberFieldValue('collection_userrating_max'),
             "maximumRomsPerPlatform": GetNumberFieldValue('collection_maxroms'),
             "maximumBytesPerPlatform": GetNumberFieldValue('collection_maxplatformsize'),
-            "maximumCollectionSizeInBytes": GetNumberFieldValue('collection_maxcollectionsize')
+            "maximumCollectionSizeInBytes": GetNumberFieldValue('collection_maxcollectionsize'),
+            "folderStructure": GetNumberFieldValue('collection_directorylayout', "Standard"),
+            "includeBIOSFiles": GetBooleanFieldValue('collection_includebios'),
+            "alwaysInclude": alwaysInclude
         }
 
+        console.log("Item: " + JSON.stringify(item));
+
         return item;
     }
 
     function GetPreview() {
         var item = GenerateCollectionItem();
 
-        console.log(JSON.stringify(item));
-
         ajaxCall(
             '/api/v1/Collections/Preview',
             'POST',
@@ -375,18 +453,34 @@
         }
     }
 
-    function GetNumberFieldValue(objectName) {
+    function GetNumberFieldValue(objectName, defaultValue) {
         var obj = document.getElementById(objectName);
         var objVal = obj.value;
 
         if (objVal) {
             return objVal;
         } else {
-            return -1;
+            if (defaultValue) {
+                return defaultValue;
+            } else {
+                return -1;
+            }
+        }
+    }
+
+    function GetBooleanFieldValue(objectName) {
+        var obj = document.getElementById(objectName);
+        var objVal = obj.value;
+
+        if (objVal == "false") {
+            return false;
+        } else {
+            return true;
         }
     }
 
     function DisplayPreview(data, targetDiv) {
+        console.log(JSON.stringify(data));
         var container = document.getElementById(targetDiv);
         container.innerHTML = '';
 
@@ -430,7 +524,48 @@
                     var gameTitleCell = document.createElement('th');
                     gameTitleCell.setAttribute('colspan', 2);
                     gameTitleCell.className = 'collections_preview_gametitlecell';
-                    gameTitleCell.innerHTML = gameItem.name;
+                    
+                    // game always include popup
+                    var gameTitleInclusion = document.createElement('select');
+                    gameTitleInclusion.id = 'collections_preview_always_' + platformItem.id + '_' + gameItem.id;
+                    gameTitleInclusion.name = 'collections_preview_always';
+                    gameTitleInclusion.setAttribute('data-platform', platformItem.id);
+                    gameTitleInclusion.setAttribute('data-game', gameItem.id);
+                    gameTitleInclusion.style.float = 'right';
+
+                    var gameTitleInclusionOptions = [
+                        {
+                            "value": "None",
+                            "text": "Automatic"
+                        },
+                        {
+                            "value": "AlwaysInclude",
+                            "text": "Always Include"
+                        },
+                        {
+                            "value": "AlwaysExclude",
+                            "text": "Always Exclude"
+                        }
+                    ];
+                    for (var i = 0; i < gameTitleInclusionOptions.length; i++) {
+                        var gameTitleInclusionOption = document.createElement('option');
+                        gameTitleInclusionOption.value = gameTitleInclusionOptions[i].value;
+                        gameTitleInclusionOption.innerHTML = gameTitleInclusionOptions[i].text;
+
+                        if (gameItem.inclusionStatus) {
+                            if (gameItem.inclusionStatus.inclusionState == gameTitleInclusionOptions[i].value) {
+                                gameTitleInclusionOption.selected = 'selected';
+                            }
+                        }
+
+                        gameTitleInclusion.appendChild(gameTitleInclusionOption);
+                    }
+                    gameTitleCell.appendChild(gameTitleInclusion);
+
+                    // game title label
+                    var gameTitleLabel = document.createElement('span');
+                    gameTitleLabel.innerHTML = gameItem.name;
+                    gameTitleCell.appendChild(gameTitleLabel);
 
                     gameItemRow.appendChild(gameTitleCell);
 
@@ -477,7 +612,7 @@
             container.appendChild(collectionTable);
 
             var collectionSize = document.getElementById('collectionedit_previewbox_size');
-            collectionSize.innerHTML = "Collection size: " + formatBytes(data.collectionProjectedSizeBytes);
+            collectionSize.innerHTML = "Estimated uncompressed collection size: " + formatBytes(data.collectionProjectedSizeBytes);
         }
     }
 
@@ -491,4 +626,32 @@
             label.innerHTML = '';
         }
     }
+
+    function DisplayDirectoryLabel() {
+        // hide all labels
+        var directoryLayoutLabels = document.getElementsByName('collection_directorylayout_label');
+        for (var i = 0; i < directoryLayoutLabels.length; i++) {
+            directoryLayoutLabels[i].style.display = 'none';
+        }
+
+        // display the label associated with the selection
+        var directoryLayoutMode = document.getElementById('collection_directorylayout').value;
+
+        var labelToDisplay = '';
+        if (directoryLayoutMode) {
+            switch(directoryLayoutMode) {
+                case "Gaseous":
+                    // standard mode
+                    labelToDisplay = 'collection_directorylayout_gaseous_label';
+                    break;
+
+                case "RetroPie":
+                    labelToDisplay = 'collection_directorylayout_retropie_label'
+                    break;
+
+            }
+            
+            document.getElementById(labelToDisplay).style.display = '';
+        }
+    }
 </script>

gaseous-server/wwwroot/pages/dialogs/platformmapedit.html

@@ -0,0 +1,388 @@
+<div style="position: absolute; top: 60px; left: 10px; right: 10px; bottom: 40px; overflow-x: scroll;">
+    <div style="width: 985px;">
+        <h3>Title Matching</h3>
+        <table style="width: 100%;">
+            <tr>
+                <td style="width: 25%; vertical-align: top;">
+                    <h4>Alternative Names</h4>
+                </td>
+                <td>
+                    <select id="mapping_edit_alternativenames" multiple="multiple" style="width: 100%;"></select>
+                </td>
+            </tr>
+            <tr>
+                <td style="width: 25%; vertical-align: top;">
+                    <h4>Supported File Extensions</h4>
+                </td>
+                <td>
+                    <select id="mapping_edit_supportedfileextensions" multiple="multiple" style="width: 100%;"></select>
+                </td>
+            </tr>
+        </table>
+
+        <h3>Collections</h3>
+        <table style="width: 100%;">
+            <tr>
+                <td style="width: 25%;">
+                    <h4>Standard Directory Naming</h4>
+                </td>
+                <td style="text-align: right;">
+                    <input id="mapping_edit_igdbslug" readonly="readonly" type="text" style="width: 98%;"/>
+                </td>
+            </tr>
+            <tr>
+                <td colspan="2"><strong>Note</strong>: Standard directory naming uses the IGDB slug for the platform is not editable.</td>
+            </tr>
+            <tr>
+                <td>
+                    <h4>RetroPie Directory Naming</h4>
+                </td>
+                <td style="text-align: right;">
+                    <input id="mapping_edit_retropie" type="text" style="width: 98%;"/>
+                </td>
+            </tr>
+        </table>
+
+        <h3>Web Emulator</h3>
+        <table style="width: 100%;">
+            <tr>
+                <td style="width: 25%;">
+                    <h4>Web Emulator</h4>
+                </td>
+                <td>
+                    <input id="mapping_edit_enablewebemulator" type="checkbox"><label for="mapping_edit_enablewebemulator" style="margin-left: 5px;">Enabled</label>
+                </td>
+            </tr>
+            <tr name="mapping_edit_webemulator">
+                <td style="width: 25%;">
+                    <h4>Engine</h4>
+                </td>
+                <td>
+                    <select id="mapping_edit_webemulatorengine" style="width: 100%;">
+                        
+                    </select>
+                </td>
+            </tr>
+            <tr name="mapping_edit_webemulator">
+                <td style="width: 25%;">
+                    <h4>Core</h4>
+                </td>
+                <td>
+                    <select id="mapping_edit_webemulatorcore" style="width: 100%;">
+
+                    </select>
+                </td>
+            </tr>
+            <tr name="mapping_edit_webemulator">
+                <td style="width: 25%;">
+                
+                </td>
+                <td id="mapping_edit_webemulatorhelp">
+                    
+                </td>
+            </tr>
+        </table>
+
+        <h3>BIOS/Firmware</h3>
+        <div id="mapping_edit_bios"></div>
+    </div>
+</div>
+
+<div style="position: absolute; height: 35px; left: 10px; right: 10px; bottom: 0px; text-align: right;">
+    <button value="Ok" onclick="SubmitMappingItem();">Ok</button>
+</div>
+
+
+
+<script type="text/javascript">
+    var modalContent = document.getElementsByClassName('modal-content');
+    if (!modalContent[0].classList.contains('collections_modal')) {
+        modalContent[0].classList.add('collections_modal');
+    }
+
+    var availableWebEmulators = [];
+
+    DisplayWebEmulatorContent(false);
+
+    ajaxCall(
+        '/api/v1/PlatformMaps/' + modalVariables,
+        'GET',
+        function (result) {
+            // set heading
+            document.getElementById('modal-heading').innerHTML = result.igdbName;
+
+            // populate page
+            $('#mapping_edit_alternativenames').select2 ({
+                tags: true,
+                tokenSeparators: [',']
+            });
+            AddTokensFromList('#mapping_edit_alternativenames', result.alternateNames);
+
+            $('#mapping_edit_supportedfileextensions').select2 ({
+                tags: true,
+                tokenSeparators: [','],
+                createTag: function (params) {
+                    if (params.term.indexOf('.') === -1) {
+                        // Return null to disable tag creation
+                        return null;
+                    }
+
+                    return {
+                        id: params.term.toUpperCase(),
+                        text: params.term.toUpperCase()
+                    }
+                }
+            });
+            AddTokensFromList('#mapping_edit_supportedfileextensions', result.extensions.supportedFileExtensions);
+
+            document.getElementById('mapping_edit_igdbslug').value = result.igdbSlug;
+            document.getElementById('mapping_edit_retropie').value = result.retroPieDirectoryName;
+
+            // set up web emulator drop downs
+            $('#mapping_edit_webemulatorengine').select2();
+            $('#mapping_edit_webemulatorcore').select2();
+
+            // start populating drop downs
+            if (result.webEmulator) {
+                if (result.webEmulator.availableWebEmulators.length > 0) {
+                    availableWebEmulators = result.webEmulator.availableWebEmulators;
+
+                    var offOption = new Option("-", "", false, false);
+                    $('#mapping_edit_webemulatorengine').append(offOption).trigger('change');
+                    for (var e = 0; e < result.webEmulator.availableWebEmulators.length; e++) {
+                        var newOption = new Option(result.webEmulator.availableWebEmulators[e].emulatorType, result.webEmulator.availableWebEmulators[e].emulatorType, false, false);
+                        $('#mapping_edit_webemulatorengine').append(newOption).trigger('change');
+                    }
+                    $('#mapping_edit_webemulatorengine').val(result.webEmulator.type);
+                    $('#mapping_edit_webemulatorengine').trigger('change');
+
+                    // select cores
+                    RenderWebEmulatorCores(result.webEmulator.core);
+
+                    if (result.webEmulator.type.length > 0) {
+                        document.getElementById('mapping_edit_enablewebemulator').checked = true;
+                    }
+                    
+                    DisplayWebEmulatorHelp(result.webEmulator.type);
+
+                    $('#mapping_edit_webemulatorengine').on('change', function(e) {
+                        RenderWebEmulatorCores();
+                    });
+                    if (result.webEmulator.type.length > 0) {
+                        DisplayWebEmulatorContent(true);
+                    } else {
+                        DisplayWebEmulatorContent(false);
+                    }
+                } else {
+                    // no emulators available
+                    DisplayWebEmulatorContent(false);
+                }
+            } else {
+                // no emulators available
+                DisplayWebEmulatorContent(false);
+            }
+
+            var biosTableHeaders = [
+                        {
+                            "name": "filename",
+                            "label": "Filename"
+                        },
+                        {
+                            "name": "description",
+                            "label": "Description"
+                        },
+                        {
+                            "name": "hash",
+                            "label": "MD5 Hash"
+                        }
+                    ];
+            document.getElementById('mapping_edit_bios').appendChild(
+                CreateEditableTable(
+                    'bios',
+                    biosTableHeaders
+                )
+            );
+
+            LoadEditableTableData('bios', biosTableHeaders, result.bios);
+        }
+    );
+
+    function RenderWebEmulatorCores(preSelectCore) {
+        var selectedEngine = document.getElementById('mapping_edit_webemulatorengine').value;
+        console.log("Engine: " + selectedEngine);
+        console.log("Preselect: " + preSelectCore);
+        console.log(JSON.stringify(availableWebEmulators));
+        $('#mapping_edit_webemulatorcore').empty().trigger("change");
+
+        // get cores for currently selected emulator
+        if (availableWebEmulators && (selectedEngine != undefined && selectedEngine != "")) {
+            if (availableWebEmulators.length > 0) {
+                var emuFound = false;
+                for (var e = 0; e < availableWebEmulators.length; e++) {
+                    if (availableWebEmulators[e].emulatorType == selectedEngine) {
+                        emuFound = true;
+                        for (var c = 0; c < availableWebEmulators[e].availableWebEmulatorCores.length; c++) {
+                            var coreName = availableWebEmulators[e].availableWebEmulatorCores[c].core;
+                            if (availableWebEmulators[e].availableWebEmulatorCores[c].alternateCoreName) {
+                                coreName += " (Maps to core: " + availableWebEmulators[e].availableWebEmulatorCores[c].alternateCoreName + ")";
+                            }
+                            if (availableWebEmulators[e].availableWebEmulatorCores[c].default == true) {
+                                coreName += " (Default)";
+                            }
+                            console.log(coreName);
+
+                            var newOption;
+                            if (availableWebEmulators[e].availableWebEmulatorCores[c].core == preSelectCore) {
+                                newOption = new Option(coreName, availableWebEmulators[e].availableWebEmulatorCores[c].core, true, true);
+                            } else {
+                                newOption = new Option(coreName, availableWebEmulators[e].availableWebEmulatorCores[c].core, false, false);
+                            }
+                            $('#mapping_edit_webemulatorcore').append(newOption).trigger('change');
+                        }
+                    }
+                }
+
+                if (emuFound == false) {
+                    var newOption = new Option("-", "", true, true);
+                    $('#mapping_edit_webemulatorcore').append(newOption).trigger('change');
+                }
+            } else {
+                var newOption = new Option("-", "", true, true);
+                $('#mapping_edit_webemulatorcore').append(newOption).trigger('change');
+            }
+        } else {
+            var newOption = new Option("-", "", true, true);
+            $('#mapping_edit_webemulatorcore').append(newOption).trigger('change');
+        }
+    }
+
+    function AddTokensFromList(selectObj, tagList) {
+        for (var i = 0; i < tagList.length; i++) {
+            var data = {
+                id: tagList[i],
+                text: tagList[i]
+            }
+
+            var newOption = new Option(data.text, data.id, true, true);
+            $(selectObj).append(newOption).trigger('change');
+        }
+    }
+
+    function SubmitMappingItem() {
+        var alternateNames = [];
+        for (var i = 0; i < document.getElementById('mapping_edit_alternativenames').childNodes.length; i++) {
+            var optionObj = document.getElementById('mapping_edit_alternativenames').childNodes[i];
+            alternateNames.push(optionObj.innerHTML);
+        }
+
+        var knownExtensions = $('#mapping_edit_supportedfileextensions').val();
+        
+        var extensions = {
+            "IGDBId": modalVariables,
+            "supportedFileExtensions": knownExtensions,
+            "uniqueFileExtensions": knownExtensions
+        };
+
+        var emulator = null;
+        if (document.getElementById('mapping_edit_enablewebemulator').checked == true) {
+            emulator = {
+                "type": document.getElementById('mapping_edit_webemulatorengine').value,
+                "core": document.getElementById('mapping_edit_webemulatorcore').value
+            };
+        }
+
+        var bios = [];
+        var biosTable = document.getElementById('EditableTable_bios');
+        // get rows
+        for (var i = 0; i < biosTable.childNodes.length; i++) {
+            var rowObj = biosTable.childNodes[i];
+            var biosObj = {};
+            var addBiosObj = false;
+
+            // get cells
+            for (var v = 0; v < rowObj.childNodes.length; v++) {
+                var cell = rowObj.childNodes[v];
+                if (cell.tagName.toLowerCase() != 'th') {
+                    // get input boxes
+                    for (var c = 0; c < cell.childNodes.length; c++) {
+                        var element = cell.childNodes[c];
+                        if (element) {
+                            if (element.getAttribute('data-cell')) {
+                                var nodeName = element.getAttribute('data-cell');
+                                biosObj[nodeName] = element.value;
+                                addBiosObj = true;
+                                
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+
+            if (addBiosObj == true) {
+                bios.push(biosObj);
+            }
+        }
+
+        var item = {
+            "igdbId": Number(modalVariables),
+            "igdbName": document.getElementById('modal-heading').innerHTML,
+            "igdbSlug": document.getElementById('mapping_edit_igdbslug').value,
+            "alternateNames": alternateNames,
+            "extensions": extensions,
+            "retroPieDirectoryName": document.getElementById('mapping_edit_retropie').value,
+            "webEmulator": emulator,
+            "bios": bios
+        };
+
+        console.log(JSON.stringify(item));
+
+        ajaxCall(
+            '/api/v1/PlatformMaps/' + modalVariables,
+            'PATCH',
+            function (result) {
+                loadPlatformMapping();
+                closeDialog();
+            },
+            function (error) {
+                console.error(JSON.stringify(error));
+            },
+            JSON.stringify(item)
+        );
+    }
+
+    $('#mapping_edit_webemulatorengine').on('select2:select', function (e) {
+        DisplayWebEmulatorHelp(e.params.data.id);
+    });
+
+    function DisplayWebEmulatorHelp(Emulator) {
+        var helpCell = document.getElementById('mapping_edit_webemulatorhelp');
+        switch (Emulator) {
+            case 'EmulatorJS':
+                helpCell.innerHTML = '<img src="/images/help.svg" class="banner_button_image banner_button_image_smaller" alt="Help" title="Help" /> See <a href="https://emulatorjs.org/docs4devs/Cores.html" target="_blank" class="romlink">https://emulatorjs.org/docs4devs/Cores.html</a> for more information regarding EmulatorJS cores.';
+                break;
+
+            default:
+                helpCell.innerHTML = '';
+                break;
+
+        }
+    }
+
+    $('#mapping_edit_enablewebemulator').change(function() {
+        DisplayWebEmulatorContent(this.checked);
+    });
+
+    function DisplayWebEmulatorContent(showContent) {
+        console.log(showContent);
+        var webEmulatorRows = document.getElementsByName('mapping_edit_webemulator');
+
+        for (var i = 0; i < webEmulatorRows.length; i++) {
+            if (showContent == true) {
+                webEmulatorRows[i].style.display = '';
+            } else {
+                webEmulatorRows[i].style.display = 'none';
+            }
+        }
+    }
+</script>

gaseous-server/wwwroot/pages/dialogs/rominfo.html

@@ -1,5 +1,6 @@
 <div id="properties_toc">
     <div id="properties_toc_general" name="properties_toc_item" onclick="SelectTab('general');">General</div>
+    <div id="properties_toc_attributes" name="properties_toc_item" onclick="SelectTab('attributes');">Attributes</div>
     <div id="properties_toc_match" name="properties_toc_item" onclick="SelectTab('match');">Title Match</div>
     <!--<div id="properties_toc_manage" name="properties_toc_item" onclick="SelectTab('manage');">Manage</div>-->
 </div>
@@ -39,8 +40,8 @@
                 <td id="rominfo_signaturematch"></td>
             </tr>
             <tr>
-                <th>Signature Source Flags</th>
-                <td id="rominfo_flags"></td>
+                <th>Signature Game Title</th>
+                <td id="rominfo_signaturetitle"></td>
             </tr>
             <tr>
                 <td colspan="2" style="text-align: right;"><button id="romDelete" class="redbutton">Delete</button></td>
@@ -48,6 +49,10 @@
         </table>
     </div>
 
+    <div id="properties_bodypanel_attributes" name="properties_tab" style="display: none;">
+        
+    </div>
+
     <div id="properties_bodypanel_match" name="properties_tab" style="display: none;">
         <table style="width: 100%;">
             <tr>
@@ -113,7 +118,6 @@
         romData = result;
         document.getElementById('modal-heading').innerHTML = result.name;
         document.getElementById('rominfo_platform').innerHTML = result.platform.name;
-        //document.getElementById('rominfo_filename').innerHTML = result.name;
         document.getElementById('rominfo_size').innerHTML = formatBytes(result.size, 2);
         document.getElementById('rominfo_type').innerHTML = getRomType(result.romType);
         document.getElementById('rominfo_mediatype').innerHTML = result.romTypeMedia;
@@ -121,10 +125,16 @@
         document.getElementById('rominfo_md5').innerHTML = result.mD5;
         document.getElementById('rominfo_sha1').innerHTML = result.shA1;
         document.getElementById('rominfo_signaturematch').innerHTML = result.source;
-        document.getElementById('rominfo_flags').innerHTML = result.flags;
+        document.getElementById('rominfo_signaturetitle').innerHTML = result.signatureSourceGameTitle;
 
         document.getElementById('properties_fixplatform').innerHTML = "<option value='" + result.platform.id + "' selected='selected'>" + result.platform.name + "</option>";
         document.getElementById('properties_fixgame').innerHTML = "<option value='" + gameData.id + "' selected='selected'>" + gameData.name + "</option>";
+
+        if (result.attributes.length > 0) {
+            document.getElementById('properties_bodypanel_attributes').appendChild(BuildAttributesTable(result.attributes, result.source));
+        } else {
+            document.getElementById('properties_toc_attributes').style.display = 'none';
+        }
     });
 
     function SelectTab(TabName) {
@@ -222,6 +232,57 @@
         });
     }
 
+    function BuildAttributesTable(attributes, sourceName) {
+        var aTable = document.createElement('table');
+        aTable.style.width = '100%';
+
+        for (var i = 0; i < attributes.length; i++) {
+            var aRow = document.createElement('tr');
+
+            var aTitleCell = document.createElement('th');
+            aTitleCell.width = "25%";
+            if (sourceName == "TOSEC") {
+                aTitleCell.innerHTML = ConvertTOSECAttributeName(attributes[i].key);
+            } else {
+                aTitleCell.innerHTML = attributes[i].key;
+            }
+            aRow.appendChild(aTitleCell);
+
+            var aValueCell = document.createElement('td');
+            aValueCell.width = "75%";
+            aValueCell.innerHTML = attributes[i].value;
+            aRow.appendChild(aValueCell);
+
+            aTable.appendChild(aRow);
+        }
+
+        return aTable;
+    }
+
+    function ConvertTOSECAttributeName(attributeName) {
+        var tosecAttributeNames = {
+            "cr": "Cracked",
+            "f" : "Fixed",
+            "h" : "Hacked",
+            "m" : "Modified",
+            "p" : "Pirated",
+            "t" : "Trained",
+            "tr": "Translated",
+            "o" : "Over Dump",
+            "u" : "Under Dump",
+            "v" : "Virus",
+            "b" : "Bad Dump",
+            "a" : "Alternate",
+            "!" : "Known Verified Dump"
+        };
+
+        if (attributeName in tosecAttributeNames) {
+            return tosecAttributeNames[attributeName];
+        } else {
+            return attributeName;
+        }
+    }
+
     SelectTab('general');
 
     document.getElementById('romDelete').setAttribute("onclick", "showSubDialog('romdelete', " + modalVariables + ");");

gaseous-server/wwwroot/pages/dialogs/upload.html

@@ -12,9 +12,20 @@
 <div>
     <div id="upload_target" class="dropzone"></div>
 </div>
+<table style="width: 100%;">
+    <tr>
+        <th style="width: 40%;">
+            Override automatic platform detection:
+        </th>
+        <td style="width: 60%;">
+            <select id="upload_platformoverride" style="width: 100%;"></select>
+        </td>
+    </tr>
+</table>
 
 <script type="text/javascript">
     document.getElementById('modal-heading').innerHTML = "Upload";
+    document.getElementById('upload_platformoverride').innerHTML = "<option value='0' selected='selected'>Automatic Platform</option>";
 
     var myDropzone = new Dropzone("div#upload_target", {
         url: "/api/v1/Roms",
@@ -69,4 +80,51 @@
         subModalContent.innerHTML = "";
         subModalVariables = null;
     }
+
+    $('#upload_platformoverride').select2({
+        minimumInputLength: 3,
+        ajax: {
+            url: '/api/v1/Search/Platform',
+            data: function (params) {
+                var query = {
+                    SearchString: params.term
+                }
+
+                // Query parameters will be ?SearchString=[term]
+                return query;
+            },
+            processResults: function (data) {
+                var arr = [];
+
+                // insert automatic detection item
+                arr.push({
+                    id: 0,
+                    text: "Automatic Platform"
+                });
+
+                for (var i = 0; i < data.length; i++) {
+                    arr.push({
+                        id: data[i].id,
+                        text: data[i].name
+                    });
+                }
+
+                return {
+                    results: arr
+                };
+
+            }
+        }
+    });
+
+    $('#upload_platformoverride').on('select2:select', function (e) {
+        var platformOverride = $('#upload_platformoverride').select2('data');
+        var queryString = '';
+        if (Number(platformOverride[0].id) != 0) {
+            queryString = "?OverridePlatformId=" + platformOverride[0].id;
+        }
+        console.log(queryString);
+
+        myDropzone.options.url = "/api/v1/Roms" + queryString;
+    });
 </script>

gaseous-server/wwwroot/pages/emulator.html

@@ -11,6 +11,7 @@
     var artworks = null;
     var artworksPosition = 0;
 
+    var emuGameTitle = '';
     var emuBios = '';
     var emuBackground = '';
 
@@ -33,6 +34,8 @@
         if (result.cover) {
             emuBackground = '/api/v1/Games/' + gameId + '/cover/image';
         }
+
+        emuGameTitle = gameData.name;
     });
 
     ajaxCall('/api/v1/Bios/' + platformId, 'GET', function (result) {

gaseous-server/wwwroot/pages/game.html

@@ -368,14 +368,18 @@
                         var platformRow = document.createElement('tr');
                         var platformHeader = document.createElement('th');
                         platformHeader.setAttribute('colspan', 6);
-                        platformHeader.innerHTML = result[i].platform.name;
+                        platformHeader.innerHTML = '<a href="#" onclick="ShowPlatformMappingDialog(' + result[i].platform.id + ');" style="float: right; text-decoration: none;" class="romlink"><img src="/images/map.svg" class="banner_button_image banner_button_image_smaller" alt="Edit platform mapping" title="Edit platform mapping" /></a><a href="#" onclick="ShowCollectionDialog(' + result[i].platform.id + ');" style="float: right; text-decoration: none;" class="romlink"><img src="/images/collections.svg" class="banner_button_image banner_button_image_smaller" alt="Add to collection" title="Add to collection" /></a>' + result[i].platform.name;
                         platformRow.appendChild(platformHeader);
                         newTable.appendChild(platformRow);
                     }
 
                     var launchButton = '';
                     if (result[i].emulator) {
-                        launchButton = '<a href="/index.html?page=emulator&engine=' + result[i].emulator.type + '&core=' + result[i].emulator.core + '&platformid=' + result[i].platform.id + '&gameid=' + gameId + '&rompath=' + encodeURIComponent('/api/v1/Games/' + gameId + '/roms/' + result[i].id + '/' + encodeURIComponent(result[i].name)) + '" class="romstart">Launch</a>';
+                        if (result[i].emulator.type) {
+                            if (result[i].emulator.type.length > 0) {
+                                launchButton = '<a href="/index.html?page=emulator&engine=' + result[i].emulator.type + '&core=' + result[i].emulator.core + '&platformid=' + result[i].platform.id + '&gameid=' + gameId + '&rompath=' + encodeURIComponent('/api/v1/Games/' + gameId + '/roms/' + result[i].id + '/' + encodeURIComponent(result[i].name)) + '" class="romstart">Launch</a>';
+                            }
+                        }
                     }
 
                     var newRow = [
@@ -666,4 +670,9 @@
 
         submodal.style.display = "none";
     }
+
+    function ShowCollectionDialog(platformId) {
+        modalVariables = platformId;
+        showSubDialog("collectionaddgame");
+    }
 </script>

gaseous-server/wwwroot/pages/settings.html

@@ -6,7 +6,9 @@
     <div id="properties_toc" class="settings_toc">
         <div class="filter_header">Settings</div>
         <div id="properties_toc_system" name="properties_toc_item" onclick="SelectTab('system');">System</div>
+        <div id="properties_toc_mapping" name="properties_toc_item" onclick="SelectTab('mapping');">Platform Mapping</div>
         <div id="properties_toc_bios" name="properties_toc_item" onclick="SelectTab('bios');">Firmware</div>
+        <div id="properties_toc_logs" name="properties_toc_item" onclick="SelectTab('logs');">Logs</div>
         <div id="properties_toc_about" name="properties_toc_item" onclick="SelectTab('about');">About</div>
     </div>
     <div id="properties_bodypanel">

gaseous-server/wwwroot/pages/settings/about.html

@@ -4,17 +4,37 @@
 
 <table style="width: 100%;">
     <tr>
-        <th>Home Page</th>
+        <th style="width: 20%;">Home Page</th>
         <td><a href="https://github.com/gaseous-project/gaseous-server" class="romlink">https://github.com/gaseous-project/gaseous-server</a></td>
     </tr>
     <tr>
         <th>Bugs and Feature Requests</th>
         <td><a href="https://github.com/gaseous-project/gaseous-server/issues" class="romlink">https://github.com/gaseous-project/gaseous-server/issues</a></td>
     </tr>
+    <tr>
+        <th>Join our Discord</th>
+        <td><a href="https://discord.gg/Nhu7wpT3k4" class="romlink">https://discord.gg/Nhu7wpT3k4</a></td>
+    </tr>
     <tr>
         <th>Server Version</th>
         <td id="settings_appversion"></td>
     </tr>
+    <tr>
+        <th>Database Schema Version</th>
+        <td id="settings_dbversion"></td>
+    </tr>
+    <tr>
+        <td colspan="2">
+            <h3>Projects That Make Gaseous Possible</h3>
+        </td>
+    </tr>
+    <tr>
+        <td style="text-align: center;"><a href="https://github.com/EmulatorJS/EmulatorJS" target="_blank"><img src="/images/EmulatorJS.png" style="height: 36px;" /></a></td>
+        <td>
+            The EmulatorJS Project<br />
+            <a href="https://github.com/EmulatorJS/EmulatorJS" target="_blank" class="romlink">https://github.com/EmulatorJS/EmulatorJS</a>
+        </td>
+    </tr>
     <tr>
         <td colspan="2">
             <h3>Data Sources</h2>
@@ -22,11 +42,12 @@
         </td>
     </tr>
     <tr>
-        <td>
+        <td style="text-align: center;">
             <a href="https://www.igdb.com/" target="_blank"><img src="/images/IGDB_logo.svg" style="filter: invert(100%); height: 36px;" /></a>
         </td>
         <td>
-            The Internet Game Database
+            The Internet Game Database<br />
+            <a href="https://www.igdb.com/" target="_blank" class="romlink">https://www.igdb.com/</a>
         </td>
     </tr>
     <tr>
@@ -35,20 +56,29 @@
         </td>
     </tr>
     <tr>
-        <td>
+        <td style="text-align: center;">
             <a href="https://www.tosecdev.org/" target="_blank"><img src="/images/TOSEC_logo.gif" style="height: 36px;" /></a>
         </td>
         <td>
-            The Old School Emulation Center
+            The Old School Emulation Center<br />
+            <a href="https://www.tosecdev.org/" target="_blank" class="romlink">https://www.tosecdev.org/</a>
+        </td>
+    </tr>
+    <tr>
+        <td style="text-align: center;">
+            <a href="https://www.progettosnaps.net/index.php" target="_blank"><img src="/images/ProgettoSnaps.gif" style="height: 36px;" /></a>
+        </td>
+        <td>
+            Progetto-Snaps<br />
+            <a href="https://www.progettosnaps.net/index.php" target="_blank" class="romlink">https://www.progettosnaps.net/index.php</a>
         </td>
     </tr>
 </table>
 
 <script type="text/javascript">
-    ajaxCall('/api/v1/System/Version', 'GET', function (result) {
-        if (result) {
-            var versionBox = document.getElementById('settings_appversion');
-            versionBox.innerHTML = result;
-        }
-    });
+    var versionBox = document.getElementById('settings_appversion');
+    versionBox.innerHTML = AppVersion;
+
+    var versionBox = document.getElementById('settings_dbversion');
+    versionBox.innerHTML = DBSchemaVersion;
 </script>

gaseous-server/wwwroot/pages/settings/bios.html

@@ -36,9 +36,17 @@
             if (result[i].available == true) {
                 availableText.innerHTML = 'Available';
                 availableText.className = 'greentext';
+
+                biosFilename = document.createElement('a');
+                biosFilename.href = '/api/v1/Bios/' + result[i].platformid + '/' + result[i].filename;
+                biosFilename.innerHTML = result[i].filename;
+                biosFilename.className = 'romlink';
             } else {
                 availableText.innerHTML = 'Unavailable';
                 availableText.className = 'redtext';
+
+                biosFilename = document.createElement('span');
+                biosFilename.innerHTML = result[i].filename;
             }
 
             var newRow = [

gaseous-server/wwwroot/pages/settings/logs.html

@@ -0,0 +1,82 @@
+<div id="gametitle">
+    <h1 id="gametitle_label">Logs</h1>
+</div>
+
+<a href="#" class="romlink" onclick="loadLogs();" style="float: right;"><img src="/images/refresh.svg" alt="Refresh" title="Refresh" class="banner_button_image" /></a>
+<table id="settings_events_table" style="width: 960px; max-width: 960px;" cellspacing="0">
+    
+</table>
+
+<div style="width: 960px; text-align: center;">
+    <button value="Load More" onclick="loadLogs(lastStartIndex, currentPage);">Load More</button>
+</div>
+
+<script type="text/javascript">
+    var lastStartIndex = 0;
+    var currentPage = 1;
+
+    function loadLogs(StartIndex, PageNumber) {
+        var apiQuery = '';
+
+        if (StartIndex && PageNumber) {
+            currentPage += 1;
+            apiQuery = '?StartIndex=' + StartIndex + '&PageNumber=' + PageNumber;
+        } else {
+            currentPage = 1;
+        }
+
+        ajaxCall(
+            '/api/v1/Logs' + apiQuery,
+            'GET',
+            function (result) {
+                var newTable = document.getElementById('settings_events_table');
+                if (currentPage == 1) {
+                    newTable.innerHTML = '';
+                
+                    newTable.appendChild(
+                        createTableRow(
+                            true, 
+                            [
+                                //'Id',
+                                ['Event Time', 'logs_table_cell_150px'],
+                                ['Severity', 'logs_table_cell_150px'],
+                                'Process',
+                                'Message'
+                            ],
+                            '',
+                            ''
+                            )
+                        );
+                }
+
+                for (var i = 0; i < result.length; i++) {
+                    lastStartIndex = result[i].id;
+
+                    var newRow = [
+                        //result[i].id,
+                        moment(result[i].eventTime).fromNow(),
+                        result[i].eventType,
+                        result[i].process,
+                        result[i].message
+                    ];
+
+                    newTable.appendChild(createTableRow(false, newRow, 'romrow logs_table_row_' + result[i].eventType, 'romcell logs_table_cell'));
+
+                    if (result[i].exceptionValue) {
+                        var exceptionString = "<h3>Exception</h3><pre class='logs_table_exception' style='width: 795px; word-wrap: break-word; overflow-wrap: break-word; overflow-y: scroll;'>" + syntaxHighlight(JSON.stringify(result[i].exceptionValue, null, 2)).replace(/\\n/g, "<br />    ") + "</pre>";
+                        var exRow = document.createElement('tr');
+                        var leadCell = document.createElement('td');
+                        exRow.appendChild(leadCell);
+                        var exCell = document.createElement('td');
+                        exCell.colSpan = '3';
+                        exCell.innerHTML = exceptionString;
+                        exRow.appendChild(exCell);
+                        newTable.appendChild(exRow);
+                    }
+                }
+            }
+        );
+    }
+
+    loadLogs();
+</script>

gaseous-server/wwwroot/pages/settings/mapping.html

@@ -0,0 +1,87 @@
+<div id="gametitle">
+    <h1 id="gametitle_label">Platform Mapping</h1>
+</div>
+
+<p>When determining the platform of a ROM or image (which is later used when determining the game title), only the "Unique File Extensions" are used. All other extensions are ignored as they will limit the ability of Gaseous to determine the game title (see <a href="https://github.com/gaseous-project/gaseous-server#game-image-title-matching" class="romlink">https://github.com/gaseous-project/gaseous-server#game-image-title-matching</a> for more information on how matching works).</p>
+
+<p>This list is pre-populated with some of the more common platforms. New platforms will appear in this list as titles are added.</p>
+
+<p><button value="Export to JSON" onclick="DownloadJSON();">Export to JSON</button><button id="importjson" value="Import JSON">Import JSON</button><button value="Reset to Default" onclick="loadPlatformMapping(true);">Reset to Default</button></p>
+
+<input id='uploadjson' type='file' name='files' hidden/>
+
+<table id="settings_mapping_table" style="width: 100%;" cellspacing="0">
+    
+</table>
+
+<script type="text/javascript">
+    function loadPlatformMapping(Overwrite) {
+        var queryString = '';
+        if (Overwrite == true) {
+            queryString = '?ResetToDefault=true';
+        }
+
+        ajaxCall(
+            '/api/v1/PlatformMaps' + queryString,
+            'GET',
+            function (result) {
+                var newTable = document.getElementById('settings_mapping_table');
+                newTable.innerHTML = '';
+                newTable.appendChild(
+                    createTableRow(
+                        true, 
+                        [ 
+                            'Platform',
+                            'Supported File Extensions',
+                            'Unique File Extensions',
+                            'Has Web Emulator'
+                        ],
+                        '',
+                        ''
+                        )
+                    );
+
+                for (var i = 0; i < result.length; i++) {
+                    var hasWebEmulator = '';
+                    if (result[i].webEmulator.type.length > 0) {
+                        hasWebEmulator = 'Yes';
+                    }
+
+                    var newRow = [
+                        '<a href="#/" onclick="ShowPlatformMappingDialog(' + result[i].igdbId + ');" class="romlink">' + result[i].igdbName + '</a>',
+                        result[i].extensions.supportedFileExtensions.join(', '),
+                        result[i].extensions.uniqueFileExtensions.join(', '),
+                        hasWebEmulator
+                    ];
+
+                    newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell logs_table_cell'));
+                }
+            }
+        );
+    }
+
+    function DownloadJSON() {
+        window.open('/api/v1/PlatformMaps', '_blank');
+    }
+
+    document.getElementById('importjson').addEventListener('click', openDialog);
+
+    function openDialog() {
+        document.getElementById('uploadjson').click();
+    }
+
+    $('#uploadjson').change(function () {
+        $(this).simpleUpload("/api/v1/PlatformMaps", {
+            start: function (file) {
+                //upload started
+                console.log("JSON upload started");
+            },
+            success: function(data){
+                //upload successful
+                window.location.reload();
+            }
+        });
+    });
+
+    loadPlatformMapping();
+</script>

gaseous-server/wwwroot/pages/settings/system.html

@@ -52,7 +52,10 @@
                             itemTypeName = "Library scan";
                             break;
                         case 'CollectionCompiler':
-                            itemTypeName = "Compress collections";
+                            itemTypeName = "Compress collection id: " + result[i].options;
+                            break;
+                        case 'BackgroundDatabaseUpgrade':
+                            itemTypeName = "Background database upgrade";
                             break;
                         default:
                             itemTypeName = result[i].itemType;
@@ -61,36 +64,48 @@
 
                     var itemStateName;
                     var itemLastStart;
-                    switch (result[i].itemState) {
-                        case 'NeverStarted':
-                            itemStateName = "Never started";
-                            itemLastStart = '-';
-                            break;
-                        case 'Stopped':
-                            itemStateName = "Stopped";
-                            itemLastStart = moment(result[i].lastRunTime).fromNow();
-                            break;
-                        case 'Running':
-                            itemStateName = "Running";
-                            itemLastStart = moment(result[i].lastRunTime).fromNow();
-                            break;
-                        default:
-                            itemStateName = "Unknown status";
-                            itemLastStart = moment(result[i].lastRunTime).fromNow();
-                            break;
+                    if (result[i].isBlocked == false) {
+                        switch (result[i].itemState) {
+                            case 'NeverStarted':
+                                itemStateName = "Never started";
+                                itemLastStart = '-';
+                                break;
+                            case 'Stopped':
+                                itemStateName = "Stopped";
+                                itemLastStart = moment(result[i].lastRunTime).fromNow();
+                                break;
+                            case 'Running':
+                                itemStateName = "Running";
+                                itemLastStart = moment(result[i].lastRunTime).fromNow();
+                                break;
+                            default:
+                                itemStateName = "Unknown status";
+                                itemLastStart = moment(result[i].lastRunTime).fromNow();
+                                break;
+                        }
+                    } else {
+                        itemStateName = "Blocked";
+                        itemLastStart = moment(result[i].lastRunTime).fromNow();
                     }
 
+                    var itemInterval = result[i].interval;
+                    var nextRunTime = moment(result[i].nextRunTime).fromNow();
                     var startButton = '';
                     if (result[i].allowManualStart == true && result[i].itemState != "Running") {
                         startButton = "<span id='startProcess' class='romstart' onclick='StartProcess(\"" + result[i].itemType + "\");'>Start</span>";
                     }
 
+                    if (result[i].allowManualStart == false && result[i].removeWhenStopped == true) {
+                        itemInterval = '';
+                        nextRunTime = '';
+                    }
+
                     var newRow = [
                         itemTypeName,
                         itemStateName,
-                        result[i].interval,
+                        itemInterval,
                         itemLastStart,
-                        moment(result[i].nextRunTime).fromNow(),
+                        nextRunTime,
                         startButton
                     ];
                     newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell'));

gaseous-server/wwwroot/scripts/main.js

@@ -259,4 +259,118 @@ function DropDownRenderGameOption(state) {
         );
     }
     return response;
+}
+
+function syntaxHighlight(json) {
+    json = json.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+    return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?|(\{|\})?|(\[|\])?\b)/g, function (match) {
+        var cls = 'number';
+        if (/^"/.test(match)) {
+            if (/:$/.test(match)) {
+                cls = 'key';
+            } else {
+                cls = 'string';
+            }
+        } else if (/true|false/.test(match)) {
+            cls = 'boolean';
+        } else if (/null/.test(match)) {
+            cls = 'null';
+        } else if (/\{|\}/.test(match)) {
+            cls = 'brace';
+        } else if (/\[|\]/.test(match)) {
+            cls = 'square';
+        }
+        return '<span class="' + cls + '">' + match + '</span>';
+    });
+}
+
+function ShowPlatformMappingDialog(platformId) {
+    showDialog('platformmapedit', platformId);
+}
+
+function CreateEditableTable(TableName, Headers) {
+    var eDiv = document.createElement('div');
+
+    var eTable = document.createElement('table');
+    eTable.id = 'EditableTable_' + TableName;
+    eTable.style.width = '100%';
+
+    var headRow = document.createElement('tr');
+    for (var i = 0; i < Headers.length; i++) {
+        var headCell = document.createElement('th');
+        headCell.id = 'EditableTable_' + TableName + '_' + Headers[i].name;
+        headCell.innerHTML = Headers[i].label;
+        headRow.appendChild(headCell);
+    }
+    eTable.appendChild(headRow);
+
+    eDiv.appendChild(eTable);
+
+    // add more button
+    var addButton = document.createElement('button');
+    addButton.value = 'Add Row';
+    addButton.innerHTML = 'Add Row';
+    
+    $(addButton).click(function() {
+        eTable.appendChild(AddEditableTableRow(Headers));
+    });
+
+    eDiv.appendChild(addButton);
+
+    return eDiv;
+}
+
+function AddEditableTableRow(Headers) {
+    var uniqueId = Math.floor(Math.random() * Date.now());
+
+    var row = document.createElement('tr');
+    row.setAttribute('id', uniqueId);
+    for (var i = 0; i < Headers.length; i++) {
+        var cell = document.createElement('td');
+
+        var input = document.createElement('input');
+        input.type = 'text';
+        input.setAttribute('data-cell', Headers[i].name);
+        input.style.width = '95%';
+
+        cell.appendChild(input);
+        row.appendChild(cell);
+    }
+
+    // delete button
+    var delButtonCell = document.createElement('td');
+    delButtonCell.style.textAlign = 'right';
+    var delButton = document.createElement('button');
+    delButton.value = 'Delete';
+    delButton.innerHTML = 'Delete';
+    delButton.setAttribute('onclick', 'document.getElementById("' + uniqueId + '").remove();');
+
+    delButtonCell.appendChild(delButton);
+    row.appendChild(delButtonCell);
+
+    return row;
+}
+
+function LoadEditableTableData(TableName, Headers, Values) {
+    var eTable = document.getElementById('EditableTable_' + TableName);
+
+    for (var i = 0; i < Values.length; i++) {
+        // get new row
+        var row = AddEditableTableRow(Headers);
+        for (var v = 0; v < row.childNodes.length; v++) {
+            // looking at the cells here
+            var cell = row.childNodes[v];
+            for (var c = 0; c < cell.childNodes.length; c++) {
+                if (cell.childNodes[c].getAttribute('data-cell')) {
+                    var nodeName = cell.childNodes[c].getAttribute('data-cell');
+                    if (Values[i][nodeName]) {
+                        row.childNodes[v].childNodes[c].value = Values[i][nodeName];
+                    }
+                    break;
+                }
+            }
+        }
+
+        eTable.appendChild(row);
+    }
 }

gaseous-server/wwwroot/styles/style.css

@@ -143,6 +143,12 @@ h3 {
     filter: invert(100%);
 }
 
+.banner_button_image_smaller {
+    height: 16px;
+    width: 16px;
+    margin-left: 5px;
+}
+
 #banner_header {
     background-color: rgba(0, 22, 56, 0.8);
     backdrop-filter: blur(8px);
@@ -877,4 +883,45 @@ button:disabled {
 
 .bgalt1 {
     background-color: transparent;;
-}
+}
+
+.logs_table_cell_150px {
+    width: 150px;
+}
+
+.logs_table_cell {
+    vertical-align: top;
+}
+
+.logs_table_row_Information:hover {
+    background: rgba(42, 41, 150, 0.3);
+}
+
+.logs_table_row_Warning:hover {
+    background: rgba(139, 150, 41, 0.3);
+}
+
+.logs_table_row_Critical:hover {
+    background: rgba(150, 41, 41, 0.3);
+}
+
+.logs_table_row_Debug:hover {
+    background: rgba(150, 41, 135, 0.3);
+}
+
+.logs_table_exception {
+    margin-right: 10px;
+    padding: 5px;
+    border-radius: 10px;
+    border-color: #383838;
+    border-width: 1px;
+    background-color: #383838;
+}
+
+.string { color: lightblue; }
+.number { color: lightblue; }
+.boolean { color: lightblue; }
+.null { color: magenta; }
+.key { color: greenyellow; }
+.brace { color: #888; }
+.square { color: #fff000; }

gaseous-signature-ingestor/Program.cs

@@ -1,285 +0,0 @@
-using System;
-using System.IO;
-using MySql.Data.MySqlClient;
-using gaseous_romsignatureobject;
-using gaseous_signature_parser.parsers;
-using gaseous_tools;
-using MySqlX.XDevAPI;
-
-// process command line
-string[] commandLineArgs = Environment.GetCommandLineArgs();
-
-string tosecXML = "";
-bool showGames = false;
-string inArgument = "";
-foreach (string commandLineArg in commandLineArgs)
-{
-    if (commandLineArg != commandLineArgs[0])
-    {
-        if (inArgument == "")
-        {
-            switch (commandLineArg.ToLower())
-            {
-                case "-tosecpath":
-                    inArgument = commandLineArg.ToLower();
-                    break;
-                case "-showgames":
-                    showGames = true;
-                    break;
-                default:
-                    break;
-            }
-        }
-        else
-        {
-            switch (inArgument)
-            {
-                case "-tosecpath":
-                    tosecXML = commandLineArg;
-                    break;
-                default:
-                    break;
-            }
-            inArgument = "";
-        }
-    }
-}
-
-// check if Config.ConfigurationPath is valid and create it if not
-if (!Directory.Exists(Config.ConfigurationPath))
-{
-    Directory.CreateDirectory(Config.ConfigurationPath);
-}
-
-// connect to database
-Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
-// initialise the db
-db.InitDB();
-
-// process provided files
-Console.WriteLine("Processing input files:");
-if (Directory.Exists(tosecXML))
-{
-    Console.WriteLine("Processing TOSEC data files", ConsoleColor.Green);
-    Console.WriteLine("");
-    Console.WriteLine("");
-
-    tosecXML = Path.GetFullPath(tosecXML);
-    string[] tosecPathContents = Directory.GetFiles(tosecXML);
-    Array.Sort(tosecPathContents);
-    int lineFileNameLength = 0;
-    int lineGameNameLength = 0;
-
-    string sql = "";
-    Dictionary<string, object> dbDict = new Dictionary<string, object>();
-    System.Data.DataTable sigDB;
-
-    for (UInt16 i = 0; i < tosecPathContents.Length; ++i)
-    {
-        string tosecXMLFile = tosecPathContents[i];
-
-        string statusOutput = (i + 1).ToString().PadLeft(7, ' ') + " / " + tosecPathContents.Length.ToString().PadLeft(7, ' ') + " : " + Path.GetFileName(tosecXMLFile);
-        Console.SetCursorPosition(0, Console.CursorTop - 2);
-        Console.WriteLine("\r " + statusOutput.PadRight(lineFileNameLength, ' ') + "\r");
-        lineFileNameLength = statusOutput.Length;
-
-        // check tosec file md5
-        Console.WriteLine("  ==> Checking input file       ");
-        Common.hashObject hashObject = new Common.hashObject(tosecXMLFile);
-        sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
-        dbDict = new Dictionary<string, object>();
-        dbDict.Add("sourcemd5", hashObject.md5hash);
-        sigDB = db.ExecuteCMD(sql, dbDict);
-
-        if (sigDB.Rows.Count == 0)
-        {
-            // start parsing file
-            Console.SetCursorPosition(0, Console.CursorTop - 1);
-            Console.WriteLine("  ==> Parsing file              ");
-            TosecParser tosecParser = new TosecParser();
-            RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile);
-
-            // store in database
-
-            // store source object
-            bool processGames = false;
-            if (tosecObject.SourceMd5 != null)
-            {
-                Console.SetCursorPosition(0, Console.CursorTop - 1);
-                Console.WriteLine("  ==> Storing file in database  ");
-
-                sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
-                dbDict = new Dictionary<string, object>();
-                dbDict.Add("name", Common.ReturnValueIfNull(tosecObject.Name, ""));
-                dbDict.Add("description", Common.ReturnValueIfNull(tosecObject.Description, ""));
-                dbDict.Add("category", Common.ReturnValueIfNull(tosecObject.Category, ""));
-                dbDict.Add("version", Common.ReturnValueIfNull(tosecObject.Version, ""));
-                dbDict.Add("author", Common.ReturnValueIfNull(tosecObject.Author, ""));
-                dbDict.Add("email", Common.ReturnValueIfNull(tosecObject.Email, ""));
-                dbDict.Add("homepage", Common.ReturnValueIfNull(tosecObject.Homepage, ""));
-                dbDict.Add("uri", Common.ReturnValueIfNull(tosecObject.Url, ""));
-                dbDict.Add("sourcetype", Common.ReturnValueIfNull(tosecObject.SourceType, ""));
-                dbDict.Add("sourcemd5", tosecObject.SourceMd5);
-                dbDict.Add("sourcesha1", tosecObject.SourceSHA1);
-
-                sigDB = db.ExecuteCMD(sql, dbDict);
-                if (sigDB.Rows.Count == 0)
-                {
-                    // entry not present, insert it
-                    sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, sourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)";
-
-                    db.ExecuteCMD(sql, dbDict);
-
-                    processGames = true;
-                }
-
-                if (processGames == true)
-                {
-                    for (int x = 0; x < tosecObject.Games.Count; ++x)
-                    {
-                        RomSignatureObject.Game gameObject = tosecObject.Games[x];
-
-                        // update display
-                        if (showGames == true)
-                        {
-                            Console.SetCursorPosition(0, Console.CursorTop - 1);
-                            string statusGameOutput = " ==> " + (x + 1).ToString().PadLeft(7, ' ') + " / " + tosecObject.Games.Count.ToString().PadLeft(7, ' ') + " : " + gameObject.Name;
-                            Console.WriteLine("\r " + statusGameOutput.PadRight(lineGameNameLength, ' ') + "\r");
-                            lineGameNameLength = statusGameOutput.Length;
-                        }
-
-                        // set up game dictionary
-                        dbDict = new Dictionary<string, object>();
-                        dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, ""));
-                        dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, ""));
-                        dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, ""));
-                        dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, ""));
-                        dbDict.Add("demo", (int)gameObject.Demo);
-                        dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, ""));
-                        dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, ""));
-                        dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, ""));
-                        dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, ""));
-                        dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, ""));
-                        dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, ""));
-                        dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, ""));
-
-                        // store platform
-                        int gameSystem = 0;
-                        if (gameObject.System != null)
-                        {
-                            sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform";
-
-                            sigDB = db.ExecuteCMD(sql, dbDict);
-                            if (sigDB.Rows.Count == 0)
-                            {
-                                // entry not present, insert it
-                                sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                                sigDB = db.ExecuteCMD(sql, dbDict);
-
-                                gameSystem = Convert.ToInt32(sigDB.Rows[0][0]);
-                            }
-                            else
-                            {
-                                gameSystem = (int)sigDB.Rows[0][0];
-                            }
-                        }
-                        dbDict.Add("systemid", gameSystem);
-
-                        // store publisher
-                        int gamePublisher = 0;
-                        if (gameObject.Publisher != null)
-                        {
-                            sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher";
-
-                            sigDB = db.ExecuteCMD(sql, dbDict);
-                            if (sigDB.Rows.Count == 0)
-                            {
-                                // entry not present, insert it
-                                sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                                sigDB = db.ExecuteCMD(sql, dbDict);
-                                gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]);
-                            }
-                            else
-                            {
-                                gamePublisher = (int)sigDB.Rows[0][0];
-                            }
-                        }
-                        dbDict.Add("publisherid", gamePublisher);
-
-                        // store game
-                        int gameId = 0;
-                        sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND PublisherId=@publisher AND SystemId=@systemid AND Country=@country AND Language=@language";
-
-                        sigDB = db.ExecuteCMD(sql, dbDict);
-                        if (sigDB.Rows.Count == 0)
-                        {
-                            // entry not present, insert it
-                            sql = "INSERT INTO Signatures_Games " +
-                                "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
-                                "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                            sigDB = db.ExecuteCMD(sql, dbDict);
-
-                            gameId = Convert.ToInt32(sigDB.Rows[0][0]);
-                        }
-                        else
-                        {
-                            gameId = (int)sigDB.Rows[0][0];
-                        }
-
-                        // store rom
-                        foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
-                        {
-                            if (romObject.Md5 != null)
-                            {
-                                int romId = 0;
-                                sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5";
-                                dbDict = new Dictionary<string, object>();
-                                dbDict.Add("gameid", gameId);
-                                dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, ""));
-                                dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, ""));
-                                dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, ""));
-                                dbDict.Add("md5", romObject.Md5);
-                                dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, ""));
-                                dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, ""));
-
-                                if (romObject.flags != null)
-                                {
-                                    if (romObject.flags.Count > 0)
-                                    {
-                                        dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags));
-                                    }
-                                    else
-                                    {
-                                        dbDict.Add("flags", "[ ]");
-                                    }
-                                }
-                                else
-                                {
-                                    dbDict.Add("flags", "[ ]");
-                                }
-                                dbDict.Add("romtype", (int)romObject.RomType);
-                                dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, ""));
-                                dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
-
-                                sigDB = db.ExecuteCMD(sql, dbDict);
-                                if (sigDB.Rows.Count == 0)
-                                {
-                                    // entry not present, insert it
-                                    sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                                    sigDB = db.ExecuteCMD(sql, dbDict);
-
-
-                                    romId = Convert.ToInt32(sigDB.Rows[0][0]);
-                                }
-                                else
-                                {
-                                    romId = (int)sigDB.Rows[0][0];
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}

gaseous-signature-ingestor/gaseous-signature-ingestor.csproj

@@ -1,20 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
-    <RootNamespace>gaseous_signature_ingestor</RootNamespace>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\gaseous-romsignatureobject\gaseous-romsignatureobject.csproj" />
-    <ProjectReference Include="..\gaseous-signature-parser\gaseous-signature-parser.csproj" />
-    <ProjectReference Include="..\gaseous-tools\gaseous-tools.csproj" />
-  </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="MySql.Data" Version="8.1.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-  </ItemGroup>
-</Project>

gaseous-signature-ingestor/gaseous-signature-ingestor.csproj.user

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <PropertyGroup Condition=" '$(RunConfiguration)' == 'Default' ">
-    <StartAction>Project</StartAction>
-    <StartArguments>-tosecpath ~/Downloads/TOSEC\ -\ DAT\ Pack\ -\ Complete\ \(3764\)\ \(TOSEC-v2023-01-23\)/TOSEC/</StartArguments>
-    <ExternalConsole>true</ExternalConsole>
-  </PropertyGroup>
-</Project>

gaseous-signature-parser/Parsers/TosecParser.cs

@@ -1,526 +0,0 @@
-using System;
-using System.Xml;
-using System.IO;
-using System.Reflection;
-using System.Security.Cryptography;
-using gaseous_romsignatureobject;
-
-namespace gaseous_signature_parser.parsers
-{
-	public class TosecParser
-	{
-		public RomSignatureObject Parse(string XMLFile)
-		{
-            // load resources
-            var assembly = Assembly.GetExecutingAssembly();
-            // load systems list
-            List<string> TOSECSystems = new List<string>();
-            var resourceName = "gaseous_signature_parser.Support.Parsers.TOSEC.Systems.txt";
-            using (Stream stream = assembly.GetManifestResourceStream(resourceName))
-                using (StreamReader reader = new StreamReader(stream))
-            {
-                TOSECSystems = reader.ReadToEnd().Split(Environment.NewLine).ToList<string>();
-            }
-            // load video list
-            List<string> TOSECVideo = new List<string>();
-            resourceName = "gaseous_signature_parser.Support.Parsers.TOSEC.Video.txt";
-            using (Stream stream = assembly.GetManifestResourceStream(resourceName))
-            using (StreamReader reader = new StreamReader(stream))
-            {
-                TOSECVideo = reader.ReadToEnd().Split(Environment.NewLine).ToList<string>();
-            }
-            // load country list
-            Dictionary<string, string> TOSECCountry = new Dictionary<string, string>();
-            resourceName = "gaseous_signature_parser.Support.Parsers.TOSEC.Country.txt";
-            using (Stream stream = assembly.GetManifestResourceStream(resourceName))
-            using (StreamReader reader = new StreamReader(stream))
-            {
-                do
-                {
-                    string[] line = reader.ReadLine().Split(",");
-                    TOSECCountry.Add(line[0], line[1]);
-                } while (reader.EndOfStream == false);
-            }
-            // load language list
-            Dictionary<string, string> TOSECLanguage = new Dictionary<string, string>();
-            resourceName = "gaseous_signature_parser.Support.Parsers.TOSEC.Language.txt";
-            using (Stream stream = assembly.GetManifestResourceStream(resourceName))
-            using (StreamReader reader = new StreamReader(stream))
-            {
-                do
-                {
-                    string[] line = reader.ReadLine().Split(",");
-                    TOSECLanguage.Add(line[0], line[1]);
-                } while (reader.EndOfStream == false);
-            }
-            // load copyright list
-            Dictionary<string, string> TOSECCopyright = new Dictionary<string, string>();
-            resourceName = "gaseous_signature_parser.Support.Parsers.TOSEC.Copyright.txt";
-            using (Stream stream = assembly.GetManifestResourceStream(resourceName))
-            using (StreamReader reader = new StreamReader(stream))
-            {
-                do
-                {
-                    string[] line = reader.ReadLine().Split(",");
-                    TOSECCopyright.Add(line[0], line[1]);
-                } while (reader.EndOfStream == false);
-            }
-            // load development status list
-            Dictionary<string, string> TOSECDevelopment = new Dictionary<string, string>();
-            resourceName = "gaseous_signature_parser.Support.Parsers.TOSEC.DevelopmentStatus.txt";
-            using (Stream stream = assembly.GetManifestResourceStream(resourceName))
-            using (StreamReader reader = new StreamReader(stream))
-            {
-                do
-                {
-                    string[] line = reader.ReadLine().Split(",");
-                    TOSECDevelopment.Add(line[0], line[1]);
-                } while (reader.EndOfStream == false);
-            }
-
-            // get hashes of TOSEC file
-            var xmlStream = File.OpenRead(XMLFile);
-
-            var md5 = MD5.Create();
-            byte[] md5HashByte = md5.ComputeHash(xmlStream);
-            string md5Hash = BitConverter.ToString(md5HashByte).Replace("-", "").ToLowerInvariant();
-
-            var sha1 = SHA1.Create();
-            byte[] sha1HashByte = sha1.ComputeHash(xmlStream);
-            string sha1Hash = BitConverter.ToString(sha1HashByte).Replace("-", "").ToLowerInvariant();
-
-            // load TOSEC file
-            XmlDocument tosecXmlDoc = new XmlDocument();
-            tosecXmlDoc.Load(XMLFile);
-
-            RomSignatureObject tosecObject = new RomSignatureObject();
-
-            // get header
-            XmlNode xmlHeader = tosecXmlDoc.DocumentElement.SelectSingleNode("/datafile/header");
-            tosecObject.SourceType = "TOSEC";
-            tosecObject.SourceMd5 = md5Hash;
-            tosecObject.SourceSHA1 = sha1Hash;
-            foreach (XmlNode childNode in xmlHeader.ChildNodes)
-            {
-                switch (childNode.Name.ToLower())
-                {
-                    case "name":
-                        tosecObject.Name = childNode.InnerText;
-                        break;
-
-                    case "description":
-                        tosecObject.Description = childNode.InnerText;
-                        break;
-
-                    case "category":
-                        tosecObject.Category = childNode.InnerText;
-                        break;
-
-                    case "version":
-                        tosecObject.Version = childNode.InnerText;
-                        break;
-
-                    case "author":
-                        tosecObject.Author = childNode.InnerText;
-                        break;
-
-                    case "email":
-                        tosecObject.Email = childNode.InnerText;
-                        break;
-
-                    case "homepage":
-                        tosecObject.Homepage = childNode.InnerText;
-                        break;
-
-                    case "url":
-                        try
-                        {
-                            tosecObject.Url = new Uri(childNode.InnerText);
-                        }
-                        catch
-                        {
-                            tosecObject.Url = null;
-                        }
-                        break;
-                }
-            }
-
-            // get games
-            tosecObject.Games = new List<RomSignatureObject.Game>();
-            XmlNodeList xmlGames = tosecXmlDoc.DocumentElement.SelectNodes("/datafile/game");
-            foreach (XmlNode xmlGame in xmlGames)
-            {
-                RomSignatureObject.Game gameObject = new RomSignatureObject.Game();
-
-                // parse game name
-                string[] gameNameTitleParts = xmlGame.Attributes["name"].Value.Split("[");
-                string gameName = gameNameTitleParts[0];
-
-                // before split, save and remove the demo tag if present
-                if (gameName.Contains("(demo) ", StringComparison.CurrentCulture))
-                {
-                    gameObject.Demo = RomSignatureObject.Game.DemoTypes.demo;
-                    gameName = gameName.Replace("(demo) ", "");
-                }
-                else if (gameName.Contains("(demo-kiosk) ", StringComparison.CurrentCulture))
-                {
-                    gameObject.Demo = RomSignatureObject.Game.DemoTypes.demo_kiosk;
-                    gameName = gameName.Replace("(demo-kiosk) ", "");
-                }
-                else if (gameName.Contains("(demo-playable) ", StringComparison.CurrentCulture))
-                {
-                    gameObject.Demo = RomSignatureObject.Game.DemoTypes.demo_playable;
-                    gameName = gameName.Replace("(demo-playable) ", "");
-                }
-                else if (gameName.Contains("(demo-rolling) ", StringComparison.CurrentCulture))
-                {
-                    gameObject.Demo = RomSignatureObject.Game.DemoTypes.demo_rolling;
-                    gameName = gameName.Replace("(demo-rolling) ", "");
-                }
-                else if (gameName.Contains("(demo-slideshow) ", StringComparison.CurrentCulture))
-                {
-                    gameObject.Demo = RomSignatureObject.Game.DemoTypes.demo_slideshow;
-                    gameName = gameName.Replace("(demo-slideshow) ", "");
-                }
-                else
-                {
-                    gameObject.Demo = RomSignatureObject.Game.DemoTypes.NotDemo;
-                }
-
-                string[] gameNameTokens = gameName.Split("(");
-                // game title should be first item
-                gameObject.Name = gameNameTokens[0].Trim();
-
-                // game year should be second item
-                if (gameNameTokens.Length >= 2)
-                {
-                    bool dateFound = false;
-
-                    // verify the value
-                    string dateToken = gameNameTokens[1].Replace(")", "");
-                    if (dateToken.Length >= 4)
-                    {
-                        // test for possible year values
-                        // first up - centuries
-                        if (dateToken == "19xx" || dateToken == "20xx")
-                        {
-                            // date is a century
-                            gameObject.Year = dateToken;
-                            dateFound = true;
-                        } else
-                        {
-                            // check for decades
-                            for (UInt16 i = 0; i < 10; i++)
-                            {
-                                if (dateToken == "19" + i + "x" || dateToken == "20" + i + "x")
-                                {
-                                    // date is a decade
-                                    gameObject.Year = dateToken;
-                                    dateFound = true;
-                                    break;
-                                }
-                            }
-
-                            if (dateFound == false)
-                            {
-                                // check if the year is a four digit number
-                                DateTime dateTime = new DateTime();
-                                if (DateTime.TryParse(string.Format("1/1/{0}", dateToken), out dateTime))
-                                {
-                                    // is a valid year!
-                                    gameObject.Year = dateToken;
-                                    dateFound = true;
-                                }
-
-                                // if we still haven't found a valid date, check if the whole string is a valid date object
-                                if (dateFound == false)
-                                {
-                                    if (DateTime.TryParse(dateToken, out dateTime))
-                                    {
-                                        // is a valid year!
-                                        gameObject.Year = dateToken;
-                                        dateFound = true;
-                                    }
-                                }
-
-                                // if we still haven't found a valid date, check if the whole string is a valid date object, but with x's
-                                // example: 19xx-12-2x
-                                if (dateFound == false)
-                                {
-                                    if (DateTime.TryParse(dateToken.Replace("x", "0"), out dateTime))
-                                    {
-                                        // is a valid year!
-                                        gameObject.Year = dateToken;
-                                        dateFound = true;
-                                    }
-                                }
-
-                                // if we still haven't found a valid date, perhaps it a year and month?
-                                // example: 19xx-12
-                                if (dateFound == false)
-                                {
-                                    if (DateTime.TryParse(dateToken.Replace("x", "0") + "-01", out dateTime))
-                                    {
-                                        // is a valid year!
-                                        gameObject.Year = dateToken;
-                                        dateFound = true;
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-                else
-                {
-                    gameObject.Year = "";
-                }
-                // game publisher should be third item
-                if (gameNameTokens.Length >= 3)
-                {
-                    gameObject.Publisher = gameNameTokens[2].Replace(")", "").Trim();
-                }
-                else
-                {
-                    gameObject.Publisher = "";
-                }
-                // process remaining tokens
-                // set default values
-                gameObject.System = tosecObject.Name.Split(" - ")[0];
-                // process title values
-                UInt16 StartToken = 0;
-                foreach (string rawToken in gameNameTokens)
-                {
-                    if (StartToken > 2)
-                    {
-                        string[] tokenSplit = rawToken.Split("[");
-
-                        // replace the extra closing bracket
-                        string token = tokenSplit[0].Replace(")", "").Trim();
-
-                        // perform tests on the token to see what it is
-                        // exclude strings that start with [ in this part
-                        if (!(token.StartsWith("[") && token.EndsWith("]")))
-                        {
-                            // check for systems
-                            if (TOSECSystems.Contains(token, StringComparer.CurrentCulture))
-                            {
-                                // this is a system token
-                                gameObject.SystemVariant = token;
-                            }
-
-                            // check for video
-                            if (TOSECVideo.Contains(token, StringComparer.CurrentCulture))
-                            {
-                                // this is a system token
-                                gameObject.Video = token;
-                            }
-
-                            // check for country
-                            if (TOSECCountry.ContainsKey(token))
-                            {
-                                gameObject.Country = token;
-                            }
-
-                            // check for language
-                            if (TOSECLanguage.ContainsKey(token))
-                            {
-                                gameObject.Language = token;
-                            }
-
-                            // check for copyright
-                            if (TOSECCopyright.ContainsKey(token))
-                            {
-                                gameObject.Copyright = token;
-                            }
-                        }
-                    }
-                    StartToken += 1;
-                }
-
-                gameObject.Roms = new List<RomSignatureObject.Game.Rom>();
-
-                // get the roms
-                string romDescription = "";
-                foreach (XmlNode xmlGameDetail in xmlGame.ChildNodes)
-                {
-                    switch (xmlGameDetail.Name.ToLower())
-                    {
-                        case "description":
-                            romDescription = xmlGameDetail.InnerText;
-                            break;
-
-                        case "rom":
-                            RomSignatureObject.Game.Rom romObject = new RomSignatureObject.Game.Rom();
-                            if (xmlGameDetail != null)
-                            {
-                                romObject.Name = xmlGameDetail.Attributes["name"]?.Value;
-                                if (xmlGameDetail.Attributes["size"]?.Value != null)
-                                {
-                                    romObject.Size = UInt64.Parse(xmlGameDetail.Attributes["size"]?.Value);
-                                }
-                                else
-                                {
-                                    romObject.Size = 0;
-                                }
-                                romObject.Crc = xmlGameDetail.Attributes["crc"]?.Value;
-                                romObject.Md5 = xmlGameDetail.Attributes["md5"]?.Value;
-                                romObject.Sha1 = xmlGameDetail.Attributes["sha1"]?.Value;
-                                romObject.SignatureSource = RomSignatureObject.Game.Rom.SignatureSourceType.TOSEC;
-
-                                // parse name
-                                string[] romNameTokens = romDescription.Split("(");
-                                foreach (string rawToken in romNameTokens)
-                                {
-                                    string[] tokenSplit = rawToken.Split("[");
-
-                                    // replace the extra closing bracket
-                                    string token = tokenSplit[0].Replace(")", "").Trim();
-
-                                    // check for copyright
-                                    if (TOSECDevelopment.ContainsKey(token))
-                                    {
-                                        romObject.DevelopmentStatus = token;
-                                    }
-
-                                    // check for media type
-                                    if (token.StartsWith("Disc") ||
-                                    token.StartsWith("Disk") ||
-                                    token.StartsWith("File") ||
-                                    token.StartsWith("Part") ||
-                                    token.StartsWith("Side") ||
-                                    token.StartsWith("Tape"))
-                                    {
-                                        string[] tokens = token.Split(" ");
-                                        switch (tokens[0])
-                                        {
-                                            case "Disc":
-                                                romObject.RomType = RomSignatureObject.Game.Rom.RomTypes.Disc;
-                                                break;
-                                            case "Disk":
-                                                romObject.RomType = RomSignatureObject.Game.Rom.RomTypes.Disk;
-                                                break;
-                                            case "File":
-                                                romObject.RomType = RomSignatureObject.Game.Rom.RomTypes.File;
-                                                break;
-                                            case "Part":
-                                                romObject.RomType = RomSignatureObject.Game.Rom.RomTypes.Part;
-                                                break;
-                                            case "Side":
-                                                romObject.RomType = RomSignatureObject.Game.Rom.RomTypes.Side;
-                                                break;
-                                            case "Tape":
-                                                romObject.RomType = RomSignatureObject.Game.Rom.RomTypes.Tape;
-                                                break;
-                                        }
-                                        romObject.RomTypeMedia = token;
-                                    }
-
-                                    // check for media label
-                                    if (token.Length > 0 &&
-                                        (token + ")") == gameNameTokens.Last() &&
-                                        (
-                                            token != romObject.RomTypeMedia &&
-                                            token != gameObject.Publisher &&
-                                            token != gameObject.SystemVariant &&
-                                            token != gameObject.Video &&
-                                            token != gameObject.Country &&
-                                            token != gameObject.Copyright &&
-                                            token != gameObject.Language &&
-                                            token != romObject.DevelopmentStatus
-                                        )
-                                       )
-                                    {
-                                        // likely the media label?
-                                        romObject.MediaLabel = token;
-                                    }
-
-                                    // process dump flags
-                                    if (rawToken.IndexOf("[") > 0)
-                                    {
-                                        // has dump flags
-                                        string rawDumpFlags = rawToken.Substring(rawToken.IndexOf("["));
-                                        string[] dumpFlags = rawDumpFlags.Split("[");
-                                        foreach (string dumpFlag in dumpFlags)
-                                        {
-                                            string dToken = dumpFlag.Replace("]", "");
-                                            if (dToken.Length > 0)
-                                            {
-                                                string[] dTokenCompare = dToken.Split(" ");
-                                                if (dTokenCompare[0].Trim().ToLower().StartsWith("a"))
-                                                {
-                                                    romObject.flags.Add(dTokenCompare[0].Trim());
-                                                }
-                                                else
-                                                {
-
-                                                    switch (dTokenCompare[0].Trim().ToLower())
-                                                    {
-                                                        case "cr":
-                                                        // cracked
-                                                        case "f":
-                                                        // fixed
-                                                        case "h":
-                                                        // hacked
-                                                        case "m":
-                                                        // modified
-                                                        case "p":
-                                                        // pirated
-                                                        case "t":
-                                                        // trained
-                                                        case "tr":
-                                                        // translated
-                                                        case "o":
-                                                        // overdump
-                                                        case "u":
-                                                        // underdump
-                                                        case "v":
-                                                        // virus
-                                                        case "b":
-                                                        // bad dump
-                                                        case "a":
-                                                        // alternate
-                                                        case "!":
-                                                            // known verified dump
-                                                            // -------------------
-                                                            romObject.flags.Add(dToken);
-                                                            break;
-                                                    }
-                                                }
-
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-
-                            gameObject.Roms.Add(romObject);
-                            break;
-                    }
-                }
-
-                // search for existing gameObject to update
-                bool existingGameFound = false;
-                foreach (RomSignatureObject.Game existingGame in tosecObject.Games)
-                {
-                    if (existingGame.Name == gameObject.Name &&
-                        existingGame.Year == gameObject.Year &&
-                        existingGame.Publisher == gameObject.Publisher &&
-                        existingGame.Country == gameObject.Country &&
-                        existingGame.Language == gameObject.Language)
-                    {
-                        existingGame.Roms.AddRange(gameObject.Roms);
-                        existingGameFound = true;
-                        break;
-                    }
-                }
-                if (existingGameFound == false)
-                {
-                    tosecObject.Games.Add(gameObject);
-                }
-            }
-
-            return tosecObject;
-        }
-	}
-}
-

gaseous-signature-parser/Support/Parsers/TOSEC/Copyright.txt

@@ -1,9 +0,0 @@
-CW,Cardware
-CW-R,Cardware-Registered
-FW,Freeware
-GW,Giftware
-GW-R,Giftware-Registered
-LW,Licenceware
-PD,Public Domain
-SW,Shareware
-SW-R,Shareware-Registered

gaseous-signature-parser/Support/Parsers/TOSEC/Country.txt

@@ -1,68 +0,0 @@
-AE,United Arab Emirates
-AL,Albania
-AS,Asia
-AT,Austria
-AU,Australia
-BA,Bosnia and Herzegovina
-BE,Belgium
-BG,Bulgaria
-BR,Brazil
-CA,Canada
-CH,Switzerland
-CL,Chile
-CN,China
-CS,Serbia and Montenegro
-CY,Cyprus
-CZ,Czech Republic
-DE,Germany
-DK,Denmark
-EE,Estonia
-EG,Egypt
-ES,Spain
-EU,Europe
-FI,Finland
-FR,France
-GB,United Kingdom
-GR,Greece
-HK,Hong Kong
-HR,Croatia
-HU,Hungary
-ID,Indonesia
-IE,Ireland
-IL,Israel
-IN,India
-IR,Iran
-IS,Iceland
-IT,Italy
-JO,Jordan
-JP,Japan
-KR,South Korea
-LT,Lithuania
-LU,Luxembourg
-LV,Latvia
-MN,Mongolia
-MX,Mexico
-MY,Malaysia
-NL,Netherlands
-NO,Norway
-NP,Nepal
-NZ,New Zealand
-OM,Oman
-PE,Peru
-PH,Philippines
-PL,Poland
-PT,Portugal
-QA,Qatar
-RO,Romania
-RU,Russia
-SE,Sweden
-SG,Singapore
-SI,Slovenia
-SK,Slovakia
-TH,Thailand
-TR,Turkey
-TW,Taiwan
-US,United States
-VN,Vietnam
-YU,Yugoslavia
-ZA,South Africa

gaseous-signature-parser/Support/Parsers/TOSEC/DevelopmentStatus.txt

@@ -1,5 +0,0 @@
-alpha,Early test build
-beta,Later; feature complete test build
-preview,Near complete build
-pre-release,Near complete build
-proto,Unreleased; prototype software

gaseous-signature-parser/Support/Parsers/TOSEC/Language.txt

@@ -1,45 +0,0 @@
-ar,Arabic
-bg,Bulgarian
-bs,Bosnian
-cs,Czech
-cy,Welsh
-da,Danish
-de,German
-el,Greek
-en,English
-eo,Esperanto
-es,Spanish
-et,Estonian
-fa,Persian
-fi,Finnish
-fr,French
-ga,Irish
-gu,Gujarati
-he,Hebrew
-hi,Hindi
-hr,Croatian
-hu,Hungarian
-is,Icelandic
-it,Italian
-ja,Japanese
-ko,Korean
-lt,Lithuanian
-lv,Latvian
-ms,Malay
-nl,Dutch
-no,Norwegian
-pl,Polish
-pt,Portuguese
-ro,Romanian
-ru,Russian
-sk,Slovakian
-sl,Slovenian
-sq,Albanian
-sr,Serbian
-sv,Swedish
-th,Thai
-tr,Turkish
-ur,Urdu
-vi,Vietnamese
-yi,Yiddish
-zh,Chinese

gaseous-signature-parser/Support/Parsers/TOSEC/Systems.txt

@@ -1,57 +0,0 @@
-+2
-+2a
-+3
-130XE
-A1000
-A1200
-A1200-A4000
-A2000
-A2000-A3000
-A2024
-A2500-A3000UX
-A3000
-A4000
-A4000T
-A500
-A500+
-A500-A1000-A2000
-A500-A1000-A2000-CDTV
-A500-A1200
-A500-A1200-A2000-A4000
-A500-A2000
-A500-A600-A2000
-A570
-A600
-A600HD
-AGA
-AGA-CD32
-Aladdin Deck Enhancer
-CD32
-CDTV
-Computrainer
-Doctor PC Jr.
-ECS
-ECS-AGA
-Executive
-Mega ST
-Mega-STE
-OCS
-OCS-AGA
-ORCH80
-Osbourne 1
-PIANO90
-PlayChoice-10
-Plus4
-Primo-A
-Primo-A64
-Primo-B
-Primo-B64
-Pro-Primo
-ST
-STE
-STE-Falcon
-TT
-TURBO-R GT
-TURBO-R ST
-VS DualSystem
-VS UniSystem

gaseous-signature-parser/Support/Parsers/TOSEC/Video.txt

@@ -1,13 +0,0 @@
-CGA
-EGA
-HGC
-MCGA
-MDA
-NTSC
-NTSC-PAL
-PAL
-PAL-60
-PAL-NTSC
-SVGA
-VGA
-XGA

gaseous-signature-parser/gaseous-signature-parser.csproj

@@ -1,38 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <RootNamespace>gaseous_signature_parser</RootNamespace>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
-    <WarningLevel>4</WarningLevel>
-  </PropertyGroup>
-  <ItemGroup>
-    <None Remove="Support\Parsers\TOSEC\Copyright.txt" />
-    <None Remove="Support\Parsers\TOSEC\Country.txt" />
-    <None Remove="Support\Parsers\TOSEC\DevelopmentStatus.txt" />
-    <None Remove="Support\Parsers\TOSEC\Language.txt" />
-    <None Remove="Support\Parsers\TOSEC\Systems.txt" />
-    <None Remove="Support\Parsers\TOSEC\Video.txt" />
-    <None Remove="Support\" />
-    <None Remove="Classes\" />
-  </ItemGroup>
-  <ItemGroup>
-    <EmbeddedResource Include="Support\Parsers\TOSEC\Copyright.txt" />
-    <EmbeddedResource Include="Support\Parsers\TOSEC\Country.txt" />
-    <EmbeddedResource Include="Support\Parsers\TOSEC\DevelopmentStatus.txt" />
-    <EmbeddedResource Include="Support\Parsers\TOSEC\Language.txt" />
-    <EmbeddedResource Include="Support\Parsers\TOSEC\Systems.txt" />
-    <EmbeddedResource Include="Support\Parsers\TOSEC\Video.txt" />
-  </ItemGroup>
-  <ItemGroup>
-    <Folder Include="Support\" />
-    <Folder Include="Classes\" />
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\gaseous-romsignatureobject\gaseous-romsignatureobject.csproj" />
-  </ItemGroup>
-</Project>

gaseous-tools/Common.cs

@@ -61,7 +61,7 @@ namespace gaseous_tools
 			{
 				get
 				{
-					return _md5hash;
+					return _md5hash.ToLower();
 				}
 				set
 				{
@@ -73,7 +73,7 @@ namespace gaseous_tools
 			{
 				get
 				{
-					return _sha1hash;
+					return _sha1hash.ToLower();
 				}
 				set
 				{
@@ -99,6 +99,11 @@ namespace gaseous_tools
             }
             return size;
         }
+
+		public static string[] SkippableFiles = {
+				".DS_STORE",
+				"desktop.ini"
+			};
     }
 }
 

gaseous-tools/Config.cs

@@ -34,6 +34,14 @@ namespace gaseous_tools
             }
         }
 
+        public static string PlatformMappingFile
+        {
+            get
+            {
+                return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".gaseous-server", "platformmap.json");
+            }
+        }
+
         public static ConfigFile.Database DatabaseConfiguration
         {
             get
@@ -74,7 +82,9 @@ namespace gaseous_tools
         {
             get
             {
-                string logPathName = Path.Combine(LogPath, "Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + ".txt");
+                string logFileExtension = "txt";
+                
+                string logPathName = Path.Combine(LogPath, "Server Log " + DateTime.Now.ToUniversalTime().ToString("yyyyMMdd") + "." + logFileExtension);
                 return logPathName;
             }
         }
@@ -404,14 +414,6 @@ namespace gaseous_tools
                     }
                 }
 
-                public string LibrarySignatureImportDirectory_TOSEC
-                {
-                    get
-                    {
-                        return Path.Combine(LibrarySignatureImportDirectory, "TOSEC");
-                    }
-                }
-
                 public void InitLibrary()
                 {
                     if (!Directory.Exists(LibraryRootDirectory)) { Directory.CreateDirectory(LibraryRootDirectory); }
@@ -423,7 +425,6 @@ namespace gaseous_tools
                     if (!Directory.Exists(LibraryTempDirectory)) { Directory.CreateDirectory(LibraryTempDirectory); }
                     if (!Directory.Exists(LibraryCollectionsDirectory)) { Directory.CreateDirectory(LibraryCollectionsDirectory); }
                     if (!Directory.Exists(LibrarySignatureImportDirectory)) { Directory.CreateDirectory(LibrarySignatureImportDirectory); }
-                    if (!Directory.Exists(LibrarySignatureImportDirectory_TOSEC)) { Directory.CreateDirectory(LibrarySignatureImportDirectory_TOSEC); }
                 }
             }
 
@@ -467,7 +468,8 @@ namespace gaseous_tools
             {
                 public bool DebugLogging = false;
 
-                public LoggingFormat LogFormat = Logging.LoggingFormat.Json;
+                // log retention in days
+                public int LogRetention = 7;
 
                 public enum LoggingFormat
                 {

gaseous-tools/Database.cs

@@ -108,14 +108,20 @@ namespace gaseous_tools
                                     Logging.Log(Logging.LogType.Information, "Database", "Schema version is " + SchemaVer);
                                     if (SchemaVer < i)
 									{
+										// run pre-upgrade code
+										gaseous_tools.DatabaseMigration.PreUpgradeScript(i, _ConnectorType);
+										
                                         // apply schema!
-                                        Logging.Log(Logging.LogType.Information, "Database", "Schema update available - applying");
+                                        Logging.Log(Logging.LogType.Information, "Database", "Updating schema to version " + i);
                                         ExecuteCMD(dbScript, dbDict);
 
 										sql = "UPDATE schema_version SET schema_version=@schemaver";
 										dbDict = new Dictionary<string, object>();
 										dbDict.Add("schemaver", i);
 										ExecuteCMD(sql, dbDict);
+
+										// run post-upgrade code
+										gaseous_tools.DatabaseMigration.PostUpgradeScript(i, _ConnectorType);
 									}
 								}
 							}
@@ -178,6 +184,28 @@ namespace gaseous_tools
             }
         }
 
+		public int GetDatabaseSchemaVersion()
+		{
+			switch (_ConnectorType)
+			{
+				case databaseType.MySql:
+					string sql = "SELECT schema_version FROM schema_version;";
+					DataTable SchemaVersion = ExecuteCMD(sql);
+					if (SchemaVersion.Rows.Count == 0)
+					{
+						return 0;
+					}
+					else
+					{
+						return (int)SchemaVersion.Rows[0][0];
+					}
+					
+				default:
+					return 0;
+
+			}
+		}
+
         public class SQLTransactionItem
         {
 			public SQLTransactionItem()
@@ -208,7 +236,7 @@ namespace gaseous_tools
 			{
 				DataTable RetTable = new DataTable();
 
-                Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database");
+                Logging.Log(Logging.LogType.Debug, "Database", "Connecting to database", null, true);
                 MySqlConnection conn = new MySqlConnection(DBConn);
 				conn.Open();
 
@@ -226,11 +254,11 @@ namespace gaseous_tools
 
 				try
 				{
-                    Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'");
+                    Logging.Log(Logging.LogType.Debug, "Database", "Executing sql: '" + SQL + "'", null, true);
 					if (Parameters.Count > 0)
 					{
 						string dictValues = string.Join(";", Parameters.Select(x => string.Join("=", x.Key, x.Value)));
-						Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues);
+						Logging.Log(Logging.LogType.Debug, "Database", "Parameters: " + dictValues, null, true);
 					}
                     RetTable.Load(cmd.ExecuteReader());
 				} catch (Exception ex) {
@@ -239,7 +267,7 @@ namespace gaseous_tools
 					Trace.WriteLine("Full exception: " + ex.ToString());
 				}
 
-				Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection");
+				Logging.Log(Logging.LogType.Debug, "Database", "Closing database connection", null, true);
 				conn.Close();
 
 				return RetTable;

gaseous-tools/Database/MySQL/gaseous-1000.sql

@@ -1,27 +1,4 @@
--- MySQL dump 10.13  Distrib 8.0.29, for macos12 (x86_64)
---
--- Host: localhost    Database: gaseous
--- ------------------------------------------------------
--- Server version	8.0.33
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!50503 SET NameS utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
-/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `AgeRating`
---
-
 DROP TABLE IF EXISTS `AgeRating`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `AgeRating` (
   `Id` bigint NOT NULL,
   `Category` int DEFAULT NULL,
@@ -33,16 +10,9 @@ CREATE TABLE `AgeRating` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `AgeRatingContentDescription`
---
+);
 
 DROP TABLE IF EXISTS `AgeRatingContentDescription`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `AgeRatingContentDescription` (
   `Id` bigint NOT NULL,
   `Category` int DEFAULT NULL,
@@ -51,16 +21,9 @@ CREATE TABLE `AgeRatingContentDescription` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `AlternativeName`
---
+);
 
 DROP TABLE IF EXISTS `AlternativeName`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `AlternativeName` (
   `Id` bigint NOT NULL,
   `Checksum` varchar(45) DEFAULT NULL,
@@ -70,16 +33,9 @@ CREATE TABLE `AlternativeName` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Artwork`
---
+);
 
 DROP TABLE IF EXISTS `Artwork`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Artwork` (
   `Id` bigint NOT NULL,
   `AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -93,16 +49,9 @@ CREATE TABLE `Artwork` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Collection`
---
+);
 
 DROP TABLE IF EXISTS `Collection`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Collection` (
   `Id` bigint NOT NULL,
   `Checksum` varchar(45) DEFAULT NULL,
@@ -115,16 +64,9 @@ CREATE TABLE `Collection` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Company`
---
+);
 
 DROP TABLE IF EXISTS `Company`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Company` (
   `Id` bigint NOT NULL,
   `ChangeDate` datetime DEFAULT NULL,
@@ -148,16 +90,9 @@ CREATE TABLE `Company` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `CompanyLogo`
---
+);
 
 DROP TABLE IF EXISTS `CompanyLogo`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `CompanyLogo` (
   `Id` bigint NOT NULL,
   `AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -170,16 +105,9 @@ CREATE TABLE `CompanyLogo` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Cover`
---
+);
 
 DROP TABLE IF EXISTS `Cover`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Cover` (
   `Id` bigint NOT NULL,
   `AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -193,16 +121,9 @@ CREATE TABLE `Cover` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `ExternalGame`
---
+);
 
 DROP TABLE IF EXISTS `ExternalGame`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `ExternalGame` (
   `Id` bigint NOT NULL,
   `Category` int DEFAULT NULL,
@@ -220,16 +141,9 @@ CREATE TABLE `ExternalGame` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Franchise`
---
+);
 
 DROP TABLE IF EXISTS `Franchise`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Franchise` (
   `Id` bigint NOT NULL,
   `Checksum` varchar(45) DEFAULT NULL,
@@ -242,16 +156,9 @@ CREATE TABLE `Franchise` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Game`
---
+);
 
 DROP TABLE IF EXISTS `Game`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Game` (
   `Id` bigint NOT NULL,
   `AgeRatings` json DEFAULT NULL,
@@ -331,16 +238,9 @@ CREATE TABLE `Game` (
   KEY `Idx_themes` ((cast(`Themes` as unsigned array))),
   KEY `Idx_vIdeos` ((cast(`Videos` as unsigned array))),
   KEY `Idx_websites` ((cast(`Websites` as unsigned array)))
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Games_Roms`
---
+);
 
 DROP TABLE IF EXISTS `Games_Roms`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Games_Roms` (
   `Id` bigint NOT NULL AUTO_INCREMENT,
   `PlatformId` bigint DEFAULT NULL,
@@ -361,16 +261,9 @@ CREATE TABLE `Games_Roms` (
   UNIQUE KEY `Id_UNIQUE` (`Id`),
   INDEX `GameId` (`GameId` ASC) VISIBLE,
   INDEX `Id_GameId` (`GameId` ASC, `Id` ASC) VISIBLE
-) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `GameVideo`
---
+);
 
 DROP TABLE IF EXISTS `GameVideo`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `GameVideo` (
   `Id` bigint NOT NULL,
   `Checksum` varchar(45) DEFAULT NULL,
@@ -380,16 +273,9 @@ CREATE TABLE `GameVideo` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Genre`
---
+);
 
 DROP TABLE IF EXISTS `Genre`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Genre` (
   `Id` bigint NOT NULL,
   `Checksum` varchar(45) DEFAULT NULL,
@@ -401,16 +287,9 @@ CREATE TABLE `Genre` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `InvolvedCompany`
---
+);
 
 DROP TABLE IF EXISTS `InvolvedCompany`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `InvolvedCompany` (
   `Id` bigint NOT NULL,
   `Checksum` varchar(45) DEFAULT NULL,
@@ -425,16 +304,9 @@ CREATE TABLE `InvolvedCompany` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Platform`
---
+);
 
 DROP TABLE IF EXISTS `Platform`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Platform` (
   `Id` bigint NOT NULL,
   `Abbreviation` varchar(45) DEFAULT NULL,
@@ -456,16 +328,9 @@ CREATE TABLE `Platform` (
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`),
   UNIQUE KEY `Id_UNIQUE` (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `PlatformLogo`
---
+);
 
 DROP TABLE IF EXISTS `PlatformLogo`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `PlatformLogo` (
   `Id` bigint NOT NULL,
   `AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -478,16 +343,9 @@ CREATE TABLE `PlatformLogo` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Platformversion`
---
+);
 
 DROP TABLE IF EXISTS `PlatformVersion`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `PlatformVersion` (
   `Id` bigint NOT NULL,
   `Checksum` varchar(45) DEFAULT NULL,
@@ -512,16 +370,9 @@ CREATE TABLE `PlatformVersion` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Screenshot`
---
+);
 
 DROP TABLE IF EXISTS `Screenshot`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Screenshot` (
   `Id` bigint NOT NULL,
   `AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -535,31 +386,17 @@ CREATE TABLE `Screenshot` (
   `dateAdded` datetime DEFAULT NULL,
   `lastUpdated` datetime DEFAULT NULL,
   PRIMARY KEY (`Id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Settings`
---
+);
 
 DROP TABLE IF EXISTS `Settings`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Settings` (
   `Setting` varchar(45) NOT NULL,
   `Value` longtext,
   PRIMARY KEY (`Setting`),
   UNIQUE KEY `Setting_UNIQUE` (`Setting`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Signatures_Games`
---
+);
 
 DROP TABLE IF EXISTS `Signatures_Games`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Signatures_Games` (
   `Id` int NOT NULL AUTO_INCREMENT,
   `Name` varchar(255) DEFAULT NULL,
@@ -578,48 +415,27 @@ CREATE TABLE `Signatures_Games` (
   KEY `publisher_Idx` (`PublisherId`),
   KEY `system_Idx` (`SystemId`),
   KEY `ingest_Idx` (`Name`,`Year`,`PublisherId`,`SystemId`,`Country`,`Language`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Signatures_Platforms`
---
+);
 
 DROP TABLE IF EXISTS `Signatures_Platforms`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Signatures_Platforms` (
   `Id` int NOT NULL AUTO_INCREMENT,
   `Platform` varchar(100) DEFAULT NULL,
   PRIMARY KEY (`Id`),
   UNIQUE KEY `IdSignatures_Platforms_UNIQUE` (`Id`),
   KEY `Platforms_Idx` (`Platform`,`Id`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Signatures_Publishers`
---
+);
 
 DROP TABLE IF EXISTS `Signatures_Publishers`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Signatures_Publishers` (
   `Id` int NOT NULL AUTO_INCREMENT,
   `Publisher` varchar(100) DEFAULT NULL,
   PRIMARY KEY (`Id`),
   UNIQUE KEY `Id_UNIQUE` (`Id`),
   KEY `publisher_Idx` (`Publisher`,`Id`)
-) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Signatures_Roms`
---
+);
 
 DROP TABLE IF EXISTS `Signatures_Roms`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Signatures_Roms` (
   `Id` int NOT NULL AUTO_INCREMENT,
   `GameId` int DEFAULT NULL,
@@ -640,16 +456,9 @@ CREATE TABLE `Signatures_Roms` (
   KEY `md5_Idx` (`MD5`) USING BTREE,
   KEY `sha1_Idx` (`SHA1`) USING BTREE,
   KEY `flags_Idx` ((cast(`Flags` as char(255) array)))
-) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `Signatures_Sources`
---
+);
 
 DROP TABLE IF EXISTS `Signatures_Sources`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!50503 SET character_set_client = utf8mb4 */;
 CREATE TABLE `Signatures_Sources` (
   `Id` int NOT NULL AUTO_INCREMENT,
   `Name` varchar(255) DEFAULT NULL,
@@ -667,20 +476,7 @@ CREATE TABLE `Signatures_Sources` (
   UNIQUE KEY `Id_UNIQUE` (`Id`),
   KEY `sourcemd5_Idx` (`SourceMD5`,`Id`) USING BTREE,
   KEY `sourcesha1_Idx` (`SourceSHA1`,`Id`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Dumping events for database 'gaseous'
---
-
---
--- Dumping routines for database 'gaseous'
---
-
---
--- Final view structure for view `view_Signatures_Games`
---
+);
 
 DROP VIEW IF EXISTS `view_Signatures_Games`;
 CREATE VIEW `view_Signatures_Games` AS

gaseous-tools/Database/MySQL/gaseous-1001.sql

@@ -1,5 +1,5 @@
 DROP TABLE IF EXISTS `GameMode`;
-CREATE TABLE `gaseous`.`GameMode` (
+CREATE TABLE `GameMode` (
   `Id` BIGINT NOT NULL,
   `CreatedAt` DATETIME NULL,
   `Checksum` VARCHAR(45) NULL,
@@ -82,5 +82,5 @@ CREATE TABLE `RomCollections` (
   `BuiltStatus` INT NULL,
   PRIMARY KEY (`Id`));
 
-ALTER TABLE `gaseous`.`Signatures_Sources` 
+ALTER TABLE `Signatures_Sources` 
 CHANGE COLUMN `Author` `Author` LONGTEXT NULL DEFAULT NULL ;

gaseous-tools/Database/MySQL/gaseous-1002.sql

@@ -0,0 +1,58 @@
+ALTER TABLE `Signatures_Roms`
+DROP INDEX `flags_Idx`;
+
+ALTER TABLE `Signatures_Roms` 
+ADD COLUMN `Attributes` JSON NULL AFTER `Flags`,
+ADD COLUMN `IngestorVersion` INT NULL DEFAULT 1;
+
+ALTER TABLE `Games_Roms` 
+ADD COLUMN `Attributes` JSON NULL AFTER `Flags`,
+ADD COLUMN `MetadataGameName` VARCHAR(255) NULL AFTER `MetadataSource`,
+ADD COLUMN `MetadataVersion` INT NULL DEFAULT 1;
+
+ALTER TABLE `RomCollections` 
+ADD COLUMN `FolderStructure` INT NULL DEFAULT 0 AFTER `MaximumCollectionSizeInBytes`,
+ADD COLUMN `IncludeBIOSFiles` BOOLEAN NULL DEFAULT 0 AFTER `FolderStructure`,
+ADD COLUMN `AlwaysInclude` JSON NULL AFTER `IncludeBIOSFiles`;
+
+CREATE TABLE `PlatformMap` (
+  `Id` BIGINT NOT NULL,
+  `RetroPieDirectoryName` VARCHAR(45) NULL,
+  `WebEmulator_Type` VARCHAR(45) NULL,
+  `WebEmulator_Core` VARCHAR(45) NULL,
+  PRIMARY KEY (`Id`),
+  UNIQUE INDEX `Id_UNIQUE` (`Id` ASC) VISIBLE);
+
+CREATE TABLE `PlatformMap_AlternateNames` (
+  `Id` BIGINT NOT NULL,
+  `Name` VARCHAR(255) NOT NULL,
+  PRIMARY KEY (`Id`, `Name`));
+
+CREATE TABLE `PlatformMap_Extensions` (
+  `Id` BIGINT NOT NULL,
+  `Extension` VARCHAR(45) NOT NULL,
+  PRIMARY KEY (`Id`, `Extension`));
+
+CREATE TABLE `PlatformMap_UniqueExtensions` (
+  `Id` BIGINT NOT NULL,
+  `Extension` VARCHAR(45) NOT NULL,
+  PRIMARY KEY (`Id`, `Extension`));
+
+CREATE TABLE `PlatformMap_Bios` (
+  `Id` BIGINT NOT NULL,
+  `Filename` VARCHAR(45) NOT NULL,
+  `Description` LONGTEXT NOT NULL,
+  `Hash` VARCHAR(45) NOT NULL,
+  PRIMARY KEY (`Id`, `Filename`, `Hash`));
+
+CREATE TABLE `ServerLogs` (
+  `Id` BIGINT NOT NULL AUTO_INCREMENT,
+  `EventTime` DATETIME NOT NULL,
+  `EventType` INT NOT NULL,
+  `Process` VARCHAR(100) NOT NULL,
+  `Message` LONGTEXT NOT NULL,
+  `Exception` LONGTEXT NULL,
+  PRIMARY KEY (`Id`));
+
+ALTER TABLE `PlatformVersion` 
+CHANGE COLUMN `PlatformLogo` `PlatformLogo` BIGINT NULL DEFAULT NULL ;

gaseous-tools/Database/MySQL/gaseous-1003.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `PlatformMap` 
+ADD COLUMN `AvailableWebEmulators` JSON NULL;

gaseous-tools/DatabaseMigration.cs

@@ -0,0 +1,140 @@
+using System;
+using System.Data;
+
+namespace gaseous_tools
+{
+	public static class DatabaseMigration
+	{
+        public static List<int> BackgroundUpgradeTargetSchemaVersions = new List<int>();
+
+        public static void PreUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType) 
+        {
+
+        }
+
+        public static void PostUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType) 
+        {
+            switch(DatabaseType)
+            {
+                case Database.databaseType.MySql:
+                    switch (TargetSchemaVersion)
+                    {
+                        case 1002:
+                            // this is a safe background task
+                            BackgroundUpgradeTargetSchemaVersions.Add(1002);
+                            break;
+                    }
+                    break;
+            }
+        }
+
+        public static void UpgradeScriptBackgroundTasks()
+        {
+            foreach (int TargetSchemaVersion in BackgroundUpgradeTargetSchemaVersions)
+            {
+                switch (TargetSchemaVersion)
+                {
+                    case 1002:
+                        MySql_1002_MigrateMetadataVersion();
+                        break;
+                }
+            }
+        }
+
+        public static void MySql_1002_MigrateMetadataVersion() {
+            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "";
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+
+            // update signature roms to v2
+            sql = "SELECT Id, Flags, Attributes, IngestorVersion FROM Signatures_Roms WHERE IngestorVersion = 1";
+            DataTable data = db.ExecuteCMD(sql);
+            if (data.Rows.Count > 0)
+            {
+                Logging.Log(Logging.LogType.Information, "Signature Ingestor - Database Update", "Updating " + data.Rows.Count + " database entries");
+                int Counter = 0;
+                int LastCounterCheck = 0;
+                foreach (DataRow row in data.Rows) 
+                {
+                    List<string> Flags = Newtonsoft.Json.JsonConvert.DeserializeObject<List<string>>((string)Common.ReturnValueIfNull(row["flags"], "[]"));
+                    List<KeyValuePair<string, object>> Attributes = new List<KeyValuePair<string, object>>();
+                    foreach (string Flag in Flags)
+                    {
+                        if (Flag.StartsWith("a"))
+                        {
+                            Attributes.Add(
+                                new KeyValuePair<string, object>(
+                                    "a",
+                                    Flag
+                                )
+                            );
+                        }
+                        else
+                        {
+                            string[] FlagCompare = Flag.Split(' ');
+                            switch (FlagCompare[0].Trim().ToLower())
+                            {
+                                case "cr":
+                                // cracked
+                                case "f":
+                                // fixed
+                                case "h":
+                                // hacked
+                                case "m":
+                                // modified
+                                case "p":
+                                // pirated
+                                case "t":
+                                // trained
+                                case "tr":
+                                // translated
+                                case "o":
+                                // overdump
+                                case "u":
+                                // underdump
+                                case "v":
+                                // virus
+                                case "b":
+                                // bad dump
+                                case "a":
+                                // alternate
+                                case "!":
+                                    // known verified dump
+                                    // -------------------
+                                    string shavedToken = Flag.Substring(FlagCompare[0].Trim().Length).Trim();
+                                    Attributes.Add(new KeyValuePair<string, object>(
+                                        FlagCompare[0].Trim().ToLower(),
+                                        shavedToken
+                                    ));
+                                    break;
+                            }
+                        }
+                    }
+
+                    string AttributesJson;
+                    if (Attributes.Count > 0)
+                    {
+                        AttributesJson = Newtonsoft.Json.JsonConvert.SerializeObject(Attributes);
+                    }
+                    else
+                    {
+                        AttributesJson = "[]";
+                    }
+
+                    string updateSQL = "UPDATE Signatures_Roms SET Attributes=@attributes, IngestorVersion=2 WHERE Id=@id";
+                    dbDict = new Dictionary<string, object>();
+                    dbDict.Add("attributes", AttributesJson);
+                    dbDict.Add("id", (int)row["Id"]);
+                    db.ExecuteCMD(updateSQL, dbDict);
+
+                    if ((Counter - LastCounterCheck) > 10) 
+                    {
+                        LastCounterCheck = Counter;
+                        Logging.Log(Logging.LogType.Information, "Signature Ingestor - Database Update", "Updating " + Counter + " / " + data.Rows.Count + " database entries");
+                    }
+                    Counter += 1;
+                }
+            }
+        }
+    }
+}

gaseous-tools/Logging.cs

@@ -1,17 +1,24 @@
 using System;
+using System.Data;
+using System.Diagnostics;
+using System.Reflection;
+using System.Reflection.Metadata.Ecma335;
+using Org.BouncyCastle.Utilities;
 namespace gaseous_tools
 {
 	public class Logging
 	{
-        static public void Log(LogType EventType, string Section, string Message, Exception? ExceptionValue = null)
+        public static bool WriteToDiskOnly { get; set; } = false;
+
+        static public void Log(LogType EventType, string ServerProcess, string Message, Exception? ExceptionValue = null, bool LogToDiskOnly = false)
         {
             LogItem logItem = new LogItem
             {
                 EventTime = DateTime.UtcNow,
                 EventType = EventType,
-                Section = Section,
+                Process = ServerProcess,
                 Message = Message,
-                ExceptionValue = ExceptionValue
+                ExceptionValue = Common.ReturnValueIfNull(ExceptionValue, "").ToString()
             };
 
             bool AllowWrite = false;
@@ -30,35 +37,117 @@ namespace gaseous_tools
             if (AllowWrite == true)
             {
                 // console output
-                string TraceOutput = logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Section + ": " + logItem.Message;
+                string TraceOutput = logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Process + ": " + logItem.Message;
                 if (logItem.ExceptionValue != null)
                 {
                     TraceOutput += Environment.NewLine + logItem.ExceptionValue.ToString();
                 }
-                Console.WriteLine(TraceOutput);
-
-                StreamWriter LogFile = File.AppendText(Config.LogFilePath);
-                switch (Config.LoggingConfiguration.LogFormat)
-                {
-                    case Config.ConfigFile.Logging.LoggingFormat.Text:
-                        LogFile.WriteLine(TraceOutput);
+                switch(logItem.EventType) {
+                    case LogType.Information:
+                        Console.ForegroundColor = ConsoleColor.Blue;
                         break;
 
-                    case Config.ConfigFile.Logging.LoggingFormat.Json:
-                        Newtonsoft.Json.JsonSerializerSettings serializerSettings = new Newtonsoft.Json.JsonSerializerSettings
-                        {
-                            NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
-                            Formatting = Newtonsoft.Json.Formatting.Indented
-                        };
-                        serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
-                        string JsonOutput = Newtonsoft.Json.JsonConvert.SerializeObject(logItem, serializerSettings);
-                        LogFile.WriteLine(JsonOutput);
+                    case LogType.Warning:
+                        Console.ForegroundColor = ConsoleColor.Yellow;
                         break;
+
+                    case LogType.Critical:
+                        Console.ForegroundColor = ConsoleColor.Red;
+                        break;
+
+                    case LogType.Debug:
+                        Console.ForegroundColor = ConsoleColor.Magenta;
+                        break;
+
+                }
+                Console.WriteLine(TraceOutput);
+                Console.ResetColor();
+
+                if (WriteToDiskOnly == true)
+                {
+                    LogToDiskOnly = true;
+                }
+
+                if (LogToDiskOnly == false)
+                {
+                    Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+                    string sql = "DELETE FROM ServerLogs WHERE EventTime < @EventRententionDate; INSERT INTO ServerLogs (EventTime, EventType, Process, Message, Exception) VALUES (@EventTime, @EventType, @Process, @Message, @Exception);";
+                    Dictionary<string, object> dbDict = new Dictionary<string, object>();
+                    dbDict.Add("EventRententionDate", DateTime.UtcNow.AddDays(Config.LoggingConfiguration.LogRetention * -1));
+                    dbDict.Add("EventTime", logItem.EventTime);
+                    dbDict.Add("EventType", logItem.EventType);
+                    dbDict.Add("Process", logItem.Process);
+                    dbDict.Add("Message", logItem.Message);
+                    dbDict.Add("Exception", Common.ReturnValueIfNull(logItem.ExceptionValue, "").ToString());
+
+                    try
+                    {
+                        db.ExecuteCMD(sql, dbDict);
+                    }
+                    catch (Exception ex)
+                    {   
+                        LogToDisk(logItem, TraceOutput, ex);
+                    }
+                }
+                else
+                {
+                    LogToDisk(logItem, TraceOutput, null);
                 }
-                LogFile.Close();
             }
         }
 
+        static void LogToDisk(LogItem logItem, string TraceOutput, Exception? exception)
+        {
+            if (exception != null)
+            {
+                // dump the error
+                File.AppendAllText(Config.LogFilePath, logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": " + logItem.EventType.ToString() + ": " + logItem.Process + ": " + logItem.Message + Environment.NewLine + exception.ToString());
+            
+
+                // something went wrong writing to the db
+                File.AppendAllText(Config.LogFilePath, logItem.EventTime.ToString("yyyyMMdd HHmmss") + ": The following event was unable to be written to the log database:");
+            }
+
+            File.AppendAllText(Config.LogFilePath, TraceOutput);
+        }
+
+        static public List<LogItem> GetLogs(long? StartIndex, int PageNumber = 1, int PageSize = 100) 
+        {
+            Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "";
+            if (StartIndex == null)
+            {
+                sql = "SELECT * FROM ServerLogs ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+            }
+            else
+            {
+                sql = "SELECT * FROM ServerLogs WHERE Id < @StartIndex ORDER BY Id DESC LIMIT @PageSize OFFSET @PageNumber;";
+            }
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+            dbDict.Add("StartIndex", StartIndex);
+            dbDict.Add("PageNumber", (PageNumber - 1) * PageSize);
+            dbDict.Add("PageSize", PageSize);
+            DataTable dataTable = db.ExecuteCMD(sql, dbDict);
+
+            List<LogItem> logs = new List<LogItem>();
+            foreach (DataRow row in dataTable.Rows)
+            {
+                LogItem log = new LogItem
+                {
+                    Id = (long)row["Id"],
+                    EventTime = DateTime.Parse(((DateTime)row["EventTime"]).ToString("yyyy-MM-ddThh:mm:ss") + 'Z'),
+                    EventType = (LogType)row["EventType"],
+                    Process = (string)row["Process"],
+                    Message = (string)row["Message"],
+                    ExceptionValue = (string)row["Exception"]
+                };
+
+                logs.Add(log);
+            }
+
+            return logs;
+        }
+
         public enum LogType
         {
             Information = 0,
@@ -69,20 +158,10 @@ namespace gaseous_tools
 
         public class LogItem
         {
+            public long Id { get; set; }
             public DateTime EventTime { get; set; }
-            public LogType EventType { get; set; }
-            private string _Section = "";
-            public string Section
-            {
-                get
-                {
-                    return _Section;
-                }
-                set
-                {
-                    _Section = value;
-                }
-            }
+            public LogType? EventType { get; set; }
+            public string Process { get; set; } = "";
             private string _Message = "";
             public string Message
             {
@@ -95,7 +174,7 @@ namespace gaseous_tools
                     _Message = value;
                 }
             }
-            public Exception? ExceptionValue { get; set; }
+            public string? ExceptionValue { get; set; }
         }
     }
 }

gaseous-tools/gaseous-tools.csproj

@@ -8,15 +8,17 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="gaseous.IGDB" Version="1.0.1" />
     <PackageReference Include="MySql.Data" Version="8.1.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="IGDB" Version="2.3.2" />
   </ItemGroup>
   <ItemGroup>
     <None Remove="Database\" />
     <None Remove="Database\MySQL\" />
     <None Remove="Database\MySQL\gaseous-1000.sql" />
     <None Remove="Database\MySQL\gaseous-1001.sql" />
+    <None Remove="Database\MySQL\gaseous-1002.sql" />
+    <None Remove="Database\MySQL\gaseous-1003.sql" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="Database\" />
@@ -25,5 +27,7 @@
   <ItemGroup>
     <EmbeddedResource Include="Database\MySQL\gaseous-1000.sql" />
     <EmbeddedResource Include="Database\MySQL\gaseous-1001.sql" />
+    <EmbeddedResource Include="Database\MySQL\gaseous-1002.sql" />
+    <EmbeddedResource Include="Database\MySQL\gaseous-1003.sql" />
   </ItemGroup>
 </Project>