Expand the dat file ingestor to handle more formats (#84)

* MAME DAT's can now be imported, beginning of DB updates

* Fixed various database bugs that occur during upgrade from earlier versions

* Removed collation and MySQL specific options from database scripts
This commit is contained in:
Michael Green
2023-09-06 07:52:11 +10:00
committed by GitHub
parent 6b391bc357
commit bd7124a5be
18 changed files with 574 additions and 554 deletions

View File

@@ -1,27 +1,4 @@
-- MySQL dump 10.13 Distrib 8.0.29, for macos12 (x86_64)
--
-- Host: localhost Database: gaseous
-- ------------------------------------------------------
-- Server version 8.0.33
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!50503 SET NameS utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `AgeRating`
--
DROP TABLE IF EXISTS `AgeRating`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `AgeRating` (
`Id` bigint NOT NULL,
`Category` int DEFAULT NULL,
@@ -33,16 +10,9 @@ CREATE TABLE `AgeRating` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `AgeRatingContentDescription`
--
);
DROP TABLE IF EXISTS `AgeRatingContentDescription`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `AgeRatingContentDescription` (
`Id` bigint NOT NULL,
`Category` int DEFAULT NULL,
@@ -51,16 +21,9 @@ CREATE TABLE `AgeRatingContentDescription` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `AlternativeName`
--
);
DROP TABLE IF EXISTS `AlternativeName`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `AlternativeName` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -70,16 +33,9 @@ CREATE TABLE `AlternativeName` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Artwork`
--
);
DROP TABLE IF EXISTS `Artwork`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Artwork` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -93,16 +49,9 @@ CREATE TABLE `Artwork` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Collection`
--
);
DROP TABLE IF EXISTS `Collection`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Collection` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -115,16 +64,9 @@ CREATE TABLE `Collection` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Company`
--
);
DROP TABLE IF EXISTS `Company`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Company` (
`Id` bigint NOT NULL,
`ChangeDate` datetime DEFAULT NULL,
@@ -148,16 +90,9 @@ CREATE TABLE `Company` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `CompanyLogo`
--
);
DROP TABLE IF EXISTS `CompanyLogo`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `CompanyLogo` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -170,16 +105,9 @@ CREATE TABLE `CompanyLogo` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Cover`
--
);
DROP TABLE IF EXISTS `Cover`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Cover` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -193,16 +121,9 @@ CREATE TABLE `Cover` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `ExternalGame`
--
);
DROP TABLE IF EXISTS `ExternalGame`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `ExternalGame` (
`Id` bigint NOT NULL,
`Category` int DEFAULT NULL,
@@ -220,16 +141,9 @@ CREATE TABLE `ExternalGame` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Franchise`
--
);
DROP TABLE IF EXISTS `Franchise`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Franchise` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -242,16 +156,9 @@ CREATE TABLE `Franchise` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Game`
--
);
DROP TABLE IF EXISTS `Game`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Game` (
`Id` bigint NOT NULL,
`AgeRatings` json DEFAULT NULL,
@@ -331,16 +238,9 @@ CREATE TABLE `Game` (
KEY `Idx_themes` ((cast(`Themes` as unsigned array))),
KEY `Idx_vIdeos` ((cast(`Videos` as unsigned array))),
KEY `Idx_websites` ((cast(`Websites` as unsigned array)))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Games_Roms`
--
);
DROP TABLE IF EXISTS `Games_Roms`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Games_Roms` (
`Id` bigint NOT NULL AUTO_INCREMENT,
`PlatformId` bigint DEFAULT NULL,
@@ -361,16 +261,9 @@ CREATE TABLE `Games_Roms` (
UNIQUE KEY `Id_UNIQUE` (`Id`),
INDEX `GameId` (`GameId` ASC) VISIBLE,
INDEX `Id_GameId` (`GameId` ASC, `Id` ASC) VISIBLE
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `GameVideo`
--
);
DROP TABLE IF EXISTS `GameVideo`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `GameVideo` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -380,16 +273,9 @@ CREATE TABLE `GameVideo` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Genre`
--
);
DROP TABLE IF EXISTS `Genre`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Genre` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -401,16 +287,9 @@ CREATE TABLE `Genre` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `InvolvedCompany`
--
);
DROP TABLE IF EXISTS `InvolvedCompany`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `InvolvedCompany` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -425,16 +304,9 @@ CREATE TABLE `InvolvedCompany` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Platform`
--
);
DROP TABLE IF EXISTS `Platform`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Platform` (
`Id` bigint NOT NULL,
`Abbreviation` varchar(45) DEFAULT NULL,
@@ -456,16 +328,9 @@ CREATE TABLE `Platform` (
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`),
UNIQUE KEY `Id_UNIQUE` (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `PlatformLogo`
--
);
DROP TABLE IF EXISTS `PlatformLogo`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `PlatformLogo` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -478,16 +343,9 @@ CREATE TABLE `PlatformLogo` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Platformversion`
--
);
DROP TABLE IF EXISTS `PlatformVersion`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `PlatformVersion` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -512,16 +370,9 @@ CREATE TABLE `PlatformVersion` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Screenshot`
--
);
DROP TABLE IF EXISTS `Screenshot`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Screenshot` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -535,31 +386,17 @@ CREATE TABLE `Screenshot` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Settings`
--
);
DROP TABLE IF EXISTS `Settings`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Settings` (
`Setting` varchar(45) NOT NULL,
`Value` longtext,
PRIMARY KEY (`Setting`),
UNIQUE KEY `Setting_UNIQUE` (`Setting`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Games`
--
);
DROP TABLE IF EXISTS `Signatures_Games`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Games` (
`Id` int NOT NULL AUTO_INCREMENT,
`Name` varchar(255) DEFAULT NULL,
@@ -578,48 +415,27 @@ CREATE TABLE `Signatures_Games` (
KEY `publisher_Idx` (`PublisherId`),
KEY `system_Idx` (`SystemId`),
KEY `ingest_Idx` (`Name`,`Year`,`PublisherId`,`SystemId`,`Country`,`Language`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Platforms`
--
);
DROP TABLE IF EXISTS `Signatures_Platforms`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Platforms` (
`Id` int NOT NULL AUTO_INCREMENT,
`Platform` varchar(100) DEFAULT NULL,
PRIMARY KEY (`Id`),
UNIQUE KEY `IdSignatures_Platforms_UNIQUE` (`Id`),
KEY `Platforms_Idx` (`Platform`,`Id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Publishers`
--
);
DROP TABLE IF EXISTS `Signatures_Publishers`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Publishers` (
`Id` int NOT NULL AUTO_INCREMENT,
`Publisher` varchar(100) DEFAULT NULL,
PRIMARY KEY (`Id`),
UNIQUE KEY `Id_UNIQUE` (`Id`),
KEY `publisher_Idx` (`Publisher`,`Id`)
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Roms`
--
);
DROP TABLE IF EXISTS `Signatures_Roms`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Roms` (
`Id` int NOT NULL AUTO_INCREMENT,
`GameId` int DEFAULT NULL,
@@ -640,16 +456,9 @@ CREATE TABLE `Signatures_Roms` (
KEY `md5_Idx` (`MD5`) USING BTREE,
KEY `sha1_Idx` (`SHA1`) USING BTREE,
KEY `flags_Idx` ((cast(`Flags` as char(255) array)))
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Sources`
--
);
DROP TABLE IF EXISTS `Signatures_Sources`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Sources` (
`Id` int NOT NULL AUTO_INCREMENT,
`Name` varchar(255) DEFAULT NULL,
@@ -667,20 +476,7 @@ CREATE TABLE `Signatures_Sources` (
UNIQUE KEY `Id_UNIQUE` (`Id`),
KEY `sourcemd5_Idx` (`SourceMD5`,`Id`) USING BTREE,
KEY `sourcesha1_Idx` (`SourceSHA1`,`Id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping events for database 'gaseous'
--
--
-- Dumping routines for database 'gaseous'
--
--
-- Final view structure for view `view_Signatures_Games`
--
);
DROP VIEW IF EXISTS `view_Signatures_Games`;
CREATE VIEW `view_Signatures_Games` AS

View File

@@ -1,5 +1,5 @@
DROP TABLE IF EXISTS `GameMode`;
CREATE TABLE `gaseous`.`GameMode` (
CREATE TABLE `GameMode` (
`Id` BIGINT NOT NULL,
`CreatedAt` DATETIME NULL,
`Checksum` VARCHAR(45) NULL,
@@ -82,5 +82,5 @@ CREATE TABLE `RomCollections` (
`BuiltStatus` INT NULL,
PRIMARY KEY (`Id`));
ALTER TABLE `gaseous`.`Signatures_Sources`
ALTER TABLE `Signatures_Sources`
CHANGE COLUMN `Author` `Author` LONGTEXT NULL DEFAULT NULL ;

View File

@@ -0,0 +1,11 @@
ALTER TABLE `Signatures_Roms`
DROP INDEX `flags_Idx`;
ALTER TABLE `Signatures_Roms`
ADD COLUMN `Attributes` JSON NULL AFTER `Flags`,
ADD COLUMN `IngestorVersion` INT NULL DEFAULT 1;
ALTER TABLE `Games_Roms`
ADD COLUMN `Attributes` JSON NULL AFTER `Flags`,
ADD COLUMN `MetadataGameName` VARCHAR(255) NULL AFTER `MetadataSource`,
ADD COLUMN `MetadataVersion` INT NULL DEFAULT 1;