This commit is contained in:
Michael Green
2025-01-09 00:48:34 +11:00
parent 7c327eeb3e
commit 6863f65640
9 changed files with 426 additions and 60 deletions

View File

@@ -648,6 +648,13 @@ namespace gaseous_server.Classes
return MetadataPath;
}
public string LibraryMetadataDirectory_Hasheous()
{
string MetadataPath = Path.Combine(LibraryMetadataDirectory, "Hasheous");
if (!Directory.Exists(MetadataPath)) { Directory.CreateDirectory(MetadataPath); }
return MetadataPath;
}
public string LibrarySignaturesDirectory
{
get
@@ -668,7 +675,6 @@ namespace gaseous_server.Classes
{
if (!Directory.Exists(LibraryRootDirectory)) { Directory.CreateDirectory(LibraryRootDirectory); }
if (!Directory.Exists(LibraryImportDirectory)) { Directory.CreateDirectory(LibraryImportDirectory); }
// if (!Directory.Exists(LibraryBIOSDirectory)) { Directory.CreateDirectory(LibraryBIOSDirectory); }
if (!Directory.Exists(LibraryFirmwareDirectory)) { Directory.CreateDirectory(LibraryFirmwareDirectory); }
if (!Directory.Exists(LibraryUploadDirectory)) { Directory.CreateDirectory(LibraryUploadDirectory); }
if (!Directory.Exists(LibraryMetadataDirectory)) { Directory.CreateDirectory(LibraryMetadataDirectory); }

View File

@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System.Configuration;
using System.IO.Compression;
using System.Net;
using gaseous_server.Classes.Metadata;
@@ -311,11 +312,55 @@ namespace gaseous_server.Classes
HasheousClient.Models.LookupItemModel? HasheousResult = null;
try
{
HasheousResult = hasheous.RetrieveFromHasheous(new HasheousClient.Models.HashLookupModel
// check the cache first
if (!Directory.Exists(Config.LibraryConfiguration.LibraryMetadataDirectory_Hasheous()))
{
MD5 = hash.md5hash,
SHA1 = hash.sha1hash
});
Directory.CreateDirectory(Config.LibraryConfiguration.LibraryMetadataDirectory_Hasheous());
}
// create file name from hash object
string cacheFileName = hash.md5hash + "_" + hash.sha1hash + ".json";
string cacheFilePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Hasheous(), cacheFileName);
// use cache file if it exists and is less than 30 days old, otherwise fetch from hasheous. if the fetch from hasheous is successful, save it to the cache, if it fails, use the cache if it exists even if it's old
if (File.Exists(cacheFilePath))
{
FileInfo cacheFile = new FileInfo(cacheFilePath);
if (cacheFile.LastWriteTimeUtc > DateTime.UtcNow.AddDays(-30))
{
Logging.Log(Logging.LogType.Information, "Get Signature", "Using cached signature from Hasheous");
HasheousResult = Newtonsoft.Json.JsonConvert.DeserializeObject<HasheousClient.Models.LookupItemModel>(File.ReadAllText(cacheFilePath));
}
}
try
{
if (HasheousResult == null)
{
// fetch from hasheous
HasheousResult = hasheous.RetrieveFromHasheous(new HasheousClient.Models.HashLookupModel
{
MD5 = hash.md5hash,
SHA1 = hash.sha1hash
});
if (HasheousResult != null)
{
// save to cache
File.WriteAllText(cacheFilePath, Newtonsoft.Json.JsonConvert.SerializeObject(HasheousResult));
}
}
}
catch (Exception ex)
{
if (File.Exists(cacheFilePath))
{
Logging.Log(Logging.LogType.Warning, "Get Signature", "Error retrieving signature from Hasheous - using cached signature", ex);
HasheousResult = Newtonsoft.Json.JsonConvert.DeserializeObject<HasheousClient.Models.LookupItemModel>(File.ReadAllText(cacheFilePath));
}
else
{
Logging.Log(Logging.LogType.Warning, "Get Signature", "Error retrieving signature from Hasheous", ex);
}
}
if (HasheousResult != null)
{
@@ -366,24 +411,36 @@ namespace gaseous_server.Classes
{
foreach (HasheousClient.Models.MetadataItem metadataResult in HasheousResult.Metadata)
{
// only IGDB metadata is supported
if (metadataResult.Source == HasheousClient.Models.MetadataSources.IGDB)
if (metadataResult.ImmutableId.Length > 0)
{
if (metadataResult.ImmutableId.Length > 0)
signature.MetadataSources.AddGame(long.Parse(metadataResult.ImmutableId), HasheousResult.Name, metadataResult.Source);
}
else if (metadataResult.Id.Length > 0)
{
switch (metadataResult.Source)
{
signature.MetadataSources.AddGame(long.Parse(metadataResult.ImmutableId), HasheousResult.Name, metadataResult.Source);
}
else if (metadataResult.Id.Length > 0)
{
gaseous_server.Models.Game hasheousGame = Games.GetGame(HasheousClient.Models.MetadataSources.IGDB, metadataResult.Id);
signature.MetadataSources.AddGame((long)hasheousGame.Id, hasheousGame.Name, metadataResult.Source);
}
else
{
// no id or immutable id - use unknown game
signature.MetadataSources.AddGame(0, "Unknown Game", HasheousClient.Models.MetadataSources.None);
case HasheousClient.Models.MetadataSources.IGDB:
gaseous_server.Models.Game hasheousGame = Games.GetGame(HasheousClient.Models.MetadataSources.IGDB, metadataResult.Id);
signature.MetadataSources.AddGame((long)hasheousGame.Id, hasheousGame.Name, metadataResult.Source);
break;
default:
if (long.TryParse(metadataResult.Id, out long id) == true)
{
signature.MetadataSources.AddGame(id, HasheousResult.Name, metadataResult.Source);
}
else
{
signature.MetadataSources.AddGame(0, "Unknown Game", HasheousClient.Models.MetadataSources.None);
}
break;
}
}
else
{
// no id or immutable id - use unknown game
signature.MetadataSources.AddGame(0, "Unknown Game", HasheousClient.Models.MetadataSources.None);
}
}
}
}
@@ -421,18 +478,20 @@ namespace gaseous_server.Classes
else
{
Logging.Log(Logging.LogType.Warning, "Get Signature", "Error retrieving signature from Hasheous", ex);
throw;
}
}
else
{
Logging.Log(Logging.LogType.Warning, "Get Signature", "Error retrieving signature from Hasheous", ex);
throw;
}
}
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Warning, "Get Signature", "Error retrieving signature from Hasheous", ex);
throw;
}
}

View File

@@ -215,42 +215,42 @@ namespace gaseous_server.Classes
}
// populate map with the sources from the signature if they don't already exist
bool reloadMap = false;
foreach (MetadataSources source in Enum.GetValues(typeof(MetadataSources)))
{
bool sourceExists = false;
if (source != MetadataSources.None)
{
// check the signature for the source, and if it exists, add it to the map if it's not already there
foreach (Signatures_Games.SourceValues.SourceValueItem signatureSource in signature.MetadataSources.Games)
// get the signature that matches this source
Signatures_Games.SourceValues.SourceValueItem? signatureSource = signature.MetadataSources.Games.Find(x => x.Source == source);
if (signatureSource == null)
{
// check if the metadata map contains the source
bool sourceExists = false;
foreach (MetadataMap.MetadataMapItem mapSource in map.MetadataMapItems)
{
if (mapSource.SourceType == source)
{
sourceExists = true;
}
}
Logging.Log(Logging.LogType.Information, "Import Game", " No source found for " + source.ToString());
continue;
}
if (sourceExists == false)
// get the metadata map for this source
MetadataMap.MetadataMapItem? mapSource = map.MetadataMapItems.Find(x => x.SourceType == source);
if (mapSource == null)
{
// add the source to the map
bool preferred = false;
if (source == Config.MetadataConfiguration.DefaultMetadataSource)
{
// add the source to the map
bool preferred = false;
if (source == Config.MetadataConfiguration.DefaultMetadataSource)
{
preferred = true;
}
MetadataManagement.AddMetadataMapItem((long)map.Id, source, signatureSource.Id, preferred);
reloadMap = true;
preferred = true;
}
MetadataManagement.AddMetadataMapItem((long)map.Id, source, signatureSource.Id, preferred);
}
else
{
// update the source in the map - do not modify the preferred status
MetadataManagement.UpdateMetadataMapItem((long)map.Id, source, signatureSource.Id, null);
}
}
}
if (reloadMap == true)
{
map = MetadataManagement.GetMetadataMap((long)map.Id);
}
// reload the map
map = MetadataManagement.GetMetadataMap((long)map.Id);
// add or update the rom
dbDict = new Dictionary<string, object>();

View File

@@ -186,7 +186,7 @@ namespace gaseous_server.Classes.Metadata
if (returnObject.GetType().GetProperty("Name") != null)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT SignatureGameName FROM view_MetadataMap WHERE `Id` = @id;";
string sql = "SELECT * FROM MetadataMap JOIN MetadataMapBridge ON MetadataMap.Id = MetadataMapBridge.ParentMapId WHERE MetadataSourceId = @id AND MetadataSourceType = 0;";
DataTable dataTable = db.ExecuteCMD(sql, new Dictionary<string, object>
{
{ "@id", Id }

View File

@@ -127,6 +127,31 @@ namespace gaseous_server.Classes
db.ExecuteCMD(sql, dbDict);
}
public static void UpdateMetadataMapItem(long metadataMapId, HasheousClient.Models.MetadataSources SourceType, long sourceId, bool? preferred = null)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>()
{
{ "@metadataMapId", metadataMapId },
{ "@sourceType", SourceType },
{ "@sourceId", sourceId },
{ "@preferred", preferred }
};
if (preferred == true)
{
// set all other items to not preferred
sql = "UPDATE MetadataMapBridge SET Preferred = 0 WHERE ParentMapId = @metadataMapId; UPDATE MetadataMapBridge SET MetadataSourceId = @sourceId, Preferred = @preferred WHERE ParentMapId = @metadataMapId AND MetadataSourceType = @sourceType;";
db.ExecuteCMD(sql, dbDict);
}
else
{
sql = "UPDATE MetadataMapBridge SET MetadataSourceId = @sourceId WHERE ParentMapId = @metadataMapId AND MetadataSourceType = @sourceType;";
db.ExecuteCMD(sql, dbDict);
}
}
/// <summary>
/// Gets a metadata map from the database.
/// </summary>
@@ -333,7 +358,7 @@ namespace gaseous_server.Classes
// disabling forceRefresh
forceRefresh = false;
// update platforms
// update platform metadata
sql = "SELECT Id, `Name` FROM Platform;";
dt = db.ExecuteCMD(sql);
@@ -356,7 +381,68 @@ namespace gaseous_server.Classes
}
ClearStatus();
// update games
// update rom signatures - only valid if Haseheous is enabled
if (Config.MetadataConfiguration.SignatureSource == MetadataModel.SignatureSources.Hasheous)
{
// get all ROMs in the database
sql = "SELECT * FROM view_Games_Roms;";
dt = db.ExecuteCMD(sql);
StatusCounter = 1;
foreach (DataRow dr in dt.Rows)
{
SetStatus(StatusCounter, dt.Rows.Count, "Refreshing signature for ROM " + dr["Name"]);
try
{
Logging.Log(Logging.LogType.Information, "Metadata Refresh", "(" + StatusCounter + "/" + dt.Rows.Count + "): Refreshing signature for ROM " + dr["Name"] + " (" + dr["Id"] + ")");
// get the hash of the ROM from the datarow
string? md5 = dr["MD5"] == DBNull.Value ? null : dr["MD5"].ToString();
string? sha1 = dr["SHA1"] == DBNull.Value ? null : dr["SHA1"].ToString();
Common.hashObject hash = new Common.hashObject();
if (md5 != null)
{
hash.md5hash = md5;
}
if (sha1 != null)
{
hash.sha1hash = sha1;
}
// get the library for the ROM
GameLibrary.LibraryItem library = GameLibrary.GetLibrary((int)dr["LibraryId"]);
// get the signature for the ROM
FileInfo fi = new FileInfo(dr["Path"].ToString());
FileSignature fileSignature = new FileSignature();
gaseous_server.Models.Signatures_Games signature = fileSignature.GetFileSignature(library, hash, fi, fi.FullName);
// validate the signature - if it is invalid, skip the rest of the loop
// validation rules: 1) signature must not be null, 2) signature must have a platform ID
if (signature == null || signature.Flags.PlatformId == null)
{
Logging.Log(Logging.LogType.Information, "Metadata Refresh", "Signature for " + dr["RomName"] + " is invalid - skipping metadata refresh");
StatusCounter += 1;
continue;
}
// update the signature in the database
Platform? signaturePlatform = Metadata.Platforms.GetPlatform((long)signature.Flags.PlatformId);
ImportGame.StoreGame(library, hash, signature, signaturePlatform, fi.FullName, (long)dr["Id"], false);
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Critical, "Metadata Refresh", "An error occurred while refreshing metadata for " + dr["RomName"], ex);
}
StatusCounter += 1;
}
ClearStatus();
}
// update game metadata
if (forceRefresh == true)
{
// when forced, only update games with ROMs for

View File

@@ -318,8 +318,8 @@ namespace gaseous_server.Controllers
return NotFound();
}
string basePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(game), imageTypePath);
string imagePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(game), imageTypePath, size.ToString(), imageId + ".jpg");
string basePath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataDirectory_Game(game), imageTypePath, metadataMap.SourceType.ToString());
string imagePath = Path.Combine(basePath, size.ToString(), imageId + ".jpg");
if (!System.IO.File.Exists(imagePath))
{
@@ -796,16 +796,67 @@ namespace gaseous_server.Controllers
[MapToApiVersion("1.1")]
[HttpGet]
[Route("{MetadataMapId}/metadata")]
[ProducesResponseType(typeof(List<MetadataMap.MetadataMapItem>), StatusCodes.Status200OK)]
[ProducesResponseType(typeof(MetadataMap), StatusCodes.Status200OK)]
[ProducesResponseType(StatusCodes.Status404NotFound)]
public async Task<ActionResult> GameMetadataSources(long MetadataMapId)
{
try
{
List<MetadataMap.MetadataMapItem> metadataMapItems = Classes.MetadataManagement.GetMetadataMap(MetadataMapId).MetadataMapItems;
MetadataMap metadataMap = Classes.MetadataManagement.GetMetadataMap(MetadataMapId);
// return metadataMapItems after first removing any items where sourceType = "TheGamesDb"
return Ok(metadataMapItems.Where(x => x.SourceType != HasheousClient.Models.MetadataSources.TheGamesDb).ToList());
// return metadataMap, but filter out metadataMapItems that = "TheGamesDb"
MetadataMap filteredMetadataMap = new MetadataMap();
filteredMetadataMap.MetadataMapItems = metadataMap.MetadataMapItems.Where(x => x.SourceType != HasheousClient.Models.MetadataSources.TheGamesDb).ToList();
metadataMap.MetadataMapItems = filteredMetadataMap.MetadataMapItems;
return Ok(metadataMap);
}
catch
{
return NotFound();
}
}
[MapToApiVersion("1.0")]
[MapToApiVersion("1.1")]
[HttpPut]
[Route("{MetadataMapId}/metadata")]
[Authorize(Roles = "Admin")]
[ProducesResponseType(typeof(MetadataMap), StatusCodes.Status200OK)]
[ProducesResponseType(StatusCodes.Status404NotFound)]
public async Task<ActionResult> GameMetadataSources(long MetadataMapId, List<MetadataMap.MetadataMapItem> metadataMapItems)
{
try
{
MetadataMap existingMetadataMap = Classes.MetadataManagement.GetMetadataMap(MetadataMapId);
if (existingMetadataMap != null)
{
foreach (MetadataMap.MetadataMapItem metadataMapItem in metadataMapItems)
{
if (metadataMapItem.SourceType != HasheousClient.Models.MetadataSources.None)
{
// check if existingMetadataMap.MetadataMapItems contains metadataMapItem.SourceType
MetadataMap.MetadataMapItem existingMetadataMapItem = existingMetadataMap.MetadataMapItems.FirstOrDefault(x => x.SourceType == metadataMapItem.SourceType);
if (existingMetadataMapItem != null)
{
MetadataManagement.UpdateMetadataMapItem(MetadataMapId, metadataMapItem.SourceType, metadataMapItem.SourceId, metadataMapItem.Preferred);
}
else
{
MetadataManagement.AddMetadataMapItem(MetadataMapId, metadataMapItem.SourceType, metadataMapItem.SourceId, metadataMapItem.Preferred);
}
}
}
return Ok(Classes.MetadataManagement.GetMetadataMap(MetadataMapId));
}
else
{
return NotFound();
}
}
catch
{

View File

@@ -1,3 +1,4 @@
using gaseous_server.Classes.Metadata;
using HasheousClient.Models;
namespace gaseous_server.Models
@@ -25,6 +26,52 @@ namespace gaseous_server.Models
public HasheousClient.Models.MetadataSources SourceType { get; set; }
public long SourceId { get; set; }
public bool Preferred { get; set; }
public string SourceSlug
{
get
{
string slug = "";
switch (SourceType)
{
case MetadataSources.IGDB:
Game game = Games.GetGame(SourceType, (long)SourceId);
if (game != null)
{
slug = game.Slug;
}
break;
default:
slug = SourceId.ToString();
break;
}
return slug;
}
}
public string link
{
get
{
string link = "";
switch (SourceType)
{
case MetadataSources.IGDB:
link = $"https://www.igdb.com/games/{SourceSlug}";
break;
case MetadataSources.TheGamesDb:
link = $"https://thegamesdb.net/game.php?id={SourceId}";
break;
default:
link = "";
break;
}
return link;
}
}
}
}
}

View File

@@ -15,6 +15,7 @@ function SetupPage() {
ajaxCall('/api/v1.1/Games/' + gameId, 'GET', function (result) {
// populate games page
gameData = result;
console.log(gameData);
switch (gameData.metadataSource) {
case "IGDB":
@@ -43,7 +44,7 @@ function SetupPage() {
if (gameData.total_rating_count) {
var criticscorelabel = document.getElementById('gametitle_criticrating_label');
criticscorelabel.innerHTML = '<span style="font-size: 10px;"> User Rating<br />' + "based on " + gameData.total_rating_count + " votes</span>"
criticscorelabel.innerHTML = '<span style="font-size: 10px;"> User rating<br />based on ' + gameData.total_rating_count + ' votes</span>'
}
}
@@ -610,7 +611,116 @@ class RomManagement {
this.#SetupFixPlatformDropDown();
// add buttons
let platformEditButton = new ModalButton('Edit Platform', 0, this, async function (callingObject) {
let platformMappingButton = new ModalButton('Metadata Mapping', 0, this, async function (callingObject) {
let metadataModal = await new Modal('messagebox');
await metadataModal.BuildModal();
// override the dialog size
metadataModal.modalElement.style = 'width: 600px; height: 400px; min-width: unset; min-height: 400px; max-width: unset; max-height: 400px;';
// set the title
metadataModal.modalElement.querySelector('#modal-header-text').innerHTML = callingObject.Platform.name + ' Metadata Mapping';
// set the content
let metadataContent = metadataModal.modalElement.querySelector('#modal-body');
// fetch the metadata map
let metadataMap = await fetch('/api/v1.1/Games/' + gameId + '/metadata', {
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
}).then(response => response.json());
console.log(metadataMap);
metadataMap.metadataMapItems.forEach(element => {
let itemSection = document.createElement('div');
itemSection.className = 'section';
// header
let itemSectionHeader = document.createElement('div');
itemSectionHeader.className = 'section-header';
let itemSectionHeaderRadio = document.createElement('input');
itemSectionHeaderRadio.id = 'platformMappingSource_' + element.sourceType;
itemSectionHeaderRadio.type = 'radio';
itemSectionHeaderRadio.name = 'platformMappingSource';
itemSectionHeaderRadio.value = element.sourceType;
itemSectionHeaderRadio.style.margin = '0px';
itemSectionHeaderRadio.style.height = 'unset';
itemSectionHeaderRadio.addEventListener('change', () => {
console.log('Selected: ' + element.sourceType);
});
if (element.preferred == true) {
itemSectionHeaderRadio.checked = true;
}
itemSectionHeader.appendChild(itemSectionHeaderRadio);
let itemSectionHeaderLabel = document.createElement('label');
itemSectionHeaderLabel.htmlFor = 'platformMappingSource_' + element.sourceType;
itemSectionHeaderLabel.style.marginLeft = '10px';
itemSectionHeaderLabel.innerHTML = element.sourceType;
itemSectionHeader.appendChild(itemSectionHeaderLabel);
itemSection.appendChild(itemSectionHeader);
// content
let itemSectionContent = document.createElement('div');
itemSectionContent.className = 'section-body';
switch (element.sourceType) {
case 'None':
let noneContent = document.createElement('div');
noneContent.className = 'section-body-content';
let noneContentLabel = document.createElement('label');
noneContentLabel.innerHTML = 'No Metadata Source';
noneContent.appendChild(noneContentLabel);
itemSectionContent.appendChild(noneContent);
break;
default:
let contentLabel2 = document.createElement('div');
contentLabel2.innerHTML = 'ID: ' + element.sourceId;
itemSectionContent.appendChild(contentLabel2);
let contentLabel3 = document.createElement('div');
contentLabel3.innerHTML = 'Slug: ' + element.sourceSlug;
itemSectionContent.appendChild(contentLabel3);
if (element.link) {
if (element.link.length > 0) {
let contentLabel4 = document.createElement('div');
contentLabel4.innerHTML = 'Link: ' + element.link;
itemSectionContent.appendChild(contentLabel4);
}
}
break;
}
itemSection.appendChild(itemSectionContent);
metadataContent.appendChild(itemSection);
});
// setup the buttons
let okButton = new ModalButton('OK', 1, callingObject, async function (callingObject) {
metadataModal.close();
});
metadataModal.addButton(okButton);
let cancelButton = new ModalButton('Cancel', 0, metadataModal, async function (callingObject) {
metadataModal.close();
});
metadataModal.addButton(cancelButton);
// show the dialog
await metadataModal.open();
});
this.romsModal.addButton(platformMappingButton);
let platformEditButton = new ModalButton('Configure Emulator', 0, this, async function (callingObject) {
let mappingModal = await new Modal('messagebox');
await mappingModal.BuildModal();

View File

@@ -2923,7 +2923,7 @@ button:not(.select2-selection__choice__remove):not(.select2-selection__clear):no
display: flex;
flex-wrap: wrap;
flex-direction: row;
gap: 5px;
gap: 10px;
}
.platform_item {
@@ -2942,6 +2942,9 @@ button:not(.select2-selection__choice__remove):not(.select2-selection__clear):no
align-items: center;
position: relative;
overflow: hidden;
box-shadow: 5px 5px 19px 0px rgba(0, 0, 0, 0.44);
-webkit-box-shadow: 5px 5px 19px 0px rgba(0, 0, 0, 0.44);
-moz-box-shadow: 5px 5px 19px 0px rgba(0, 0, 0, 0.44);
}
.platform_item:hover {
@@ -2971,11 +2974,14 @@ button:not(.select2-selection__choice__remove):not(.select2-selection__clear):no
padding-left: 20px;
padding-right: 20px;
background-color: white;
border-right-style: solid;
border-right-width: 1px;
border-right-color: #000000;
}
.platform_image {
width: 70px;
max-height: 70px;
max-width: 90px;
max-height: 60px;
}
.platform_name_container {
@@ -3049,6 +3055,7 @@ button:not(.select2-selection__choice__remove):not(.select2-selection__clear):no
.metadata-attribution-icon {
width: 30px;
height: 30px;
margin-right: 5px;
margin-right: 13px;
margin-left: 10px;
filter: invert(1);
}