Fun/better-genshin-impact
1
0
Fork 0
mirror of https://github.com/babalae/better-genshin-impact.git synced 2026-05-27 10:15:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

拓展自动吃药，基于图标识别 (#2051)

Browse Source
This commit is contained in:
FishmanTheMurloc
2025-08-16 13:51:46 +08:00
committed by GitHub
parent 436a8d207e
commit 8a41148ba6
32 changed files with 674 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsConfig.cs
View File

@@ -1,4 +1,4 @@
using BetterGenshinImpact.GameTask.Model.GameUI;
using BetterGenshinImpact.GameTask.Model.GameUI;
using CommunityToolkit.Mvvm.ComponentModel;
using System;
@@ -8,8 +8,7 @@ namespace BetterGenshinImpact.GameTask.GetGridIcons;
public partial class GetGridIconsConfig : ObservableObject
{
/// <summary>
/// 昼夜策略
/// 钓全天的鱼、还是只钓白天或夜晚的鱼
/// Grid界面名称
/// </summary>
[ObservableProperty]
private GridScreenName _gridName = GridScreenName.Weapons;

45
BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsTask.cs
View File

@@ -1,17 +1,21 @@
using BetterGenshinImpact.Core.Recognition.OCR;
using BetterGenshinImpact.Core.Simulator;
using BetterGenshinImpact.GameTask.AutoArtifactSalvage;
using BetterGenshinImpact.GameTask.Common.Job;
using BetterGenshinImpact.GameTask.Model.Area;
using BetterGenshinImpact.GameTask.Model.GameUI;
using BetterGenshinImpact.Helpers.Extensions;
using Fischless.WindowsInput;
using Microsoft.Extensions.Logging;
using OpenCvSharp;
using OpenCvSharp.Extensions;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using static BetterGenshinImpact.GameTask.Common.TaskControl;
using Microsoft.Extensions.Localization;
using BetterGenshinImpact.GameTask.Model.Area;
using System.Collections.Generic;
using OpenCvSharp;
using System.Linq;
using BetterGenshinImpact.Core.Recognition.OCR;
using System.IO;
using OpenCvSharp.Extensions;
using BetterGenshinImpact.GameTask.Model.GameUI;
namespace BetterGenshinImpact.GameTask.GetGridIcons;
@@ -21,6 +25,7 @@ namespace BetterGenshinImpact.GameTask.GetGridIcons;
public class GetGridIconsTask : ISoloTask
{
private readonly ILogger logger = App.GetLogger<GetGridIconsTask>();
private readonly InputSimulator input = Simulation.SendInput;
private CancellationToken ct;
@@ -37,13 +42,31 @@ public class GetGridIconsTask : ISoloTask
this.gridScreenName = gridScreenName;
this.starAsSuffix = starAsSuffix;
this.maxNumToGet = maxNumToGet;
IStringLocalizer<GetGridIconsTask> stringLocalizer = App.GetService<IStringLocalizer<GetGridIconsTask>>() ?? throw new NullReferenceException();
}
public async Task Start(CancellationToken ct)
{
this.ct = ct;
switch (this.gridScreenName)
{
case GridScreenName.Weapons:
case GridScreenName.Artifacts:
case GridScreenName.CharacterDevelopmentItems:
case GridScreenName.Food:
case GridScreenName.Materials:
case GridScreenName.Gadget:
case GridScreenName.Quest:
case GridScreenName.PreciousItems:
case GridScreenName.Furnishings:
await new ReturnMainUiTask().Start(ct);
await AutoArtifactSalvageTask.OpenBag(this.gridScreenName, this.input, this.logger, this.ct);
break;
default:
logger.LogInformation("{name}暂不支持自动打开，请提前手动打开界面", gridScreenName.GetDescription());
break;
}
using var ra0 = CaptureToRectArea();
GridScreenParams gridParams = GridScreenParams.Templates[this.gridScreenName];
Rect gridRoi = gridParams.GetRect(ra0);

58
BetterGenshinImpact/GameTask/GetGridIcons/GridIconsAccuracyTestTask.cs
View File

@@ -1,6 +1,11 @@
using BetterGenshinImpact.Core.Recognition.OCR;
using BetterGenshinImpact.Core.Simulator;
using BetterGenshinImpact.GameTask.AutoArtifactSalvage;
using BetterGenshinImpact.GameTask.Common.Job;
using BetterGenshinImpact.GameTask.Model.Area;
using BetterGenshinImpact.GameTask.Model.GameUI;
using BetterGenshinImpact.Helpers.Extensions;
using Fischless.WindowsInput;
using Microsoft.Extensions.Logging;
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
@@ -21,6 +26,7 @@ namespace BetterGenshinImpact.GameTask.GetGridIcons;
public class GridIconsAccuracyTestTask : ISoloTask
{
private readonly ILogger logger = App.GetLogger<GetGridIconsTask>();
private readonly InputSimulator input = Simulation.SendInput;
private CancellationToken ct;
@@ -36,25 +42,28 @@ public class GridIconsAccuracyTestTask : ISoloTask
this.maxNumToTest = maxNumToTest;
}
public async Task Start(CancellationToken ct)
/// <summary>
/// 加载图标识别模型
/// </summary>
/// <param name="prototypes">原型向量</param>
/// <returns>推理会话</returns>
/// <exception cref="Exception"></exception>
public static InferenceSession LoadModel(out Dictionary<string, float[]> prototypes)
{
this.ct = ct;
#region 加载model
using var session = new InferenceSession(@".\GameTask\GetGridIcons\gridIcon.onnx"); // todo 所有数据炼好后放到onnx统一存放的位置去
var session = new InferenceSession(@".\GameTask\GetGridIcons\gridIcon.onnx"); // todo 所有数据炼好后放到onnx统一存放的位置去
var metadata = session.ModelMetadata;
if (!metadata.CustomMetadataMap.TryGetValue("prefix_list", out string? prefixListJson))
{
logger.LogError("模型文件缺少prefix_list");
return;
throw new Exception("模型文件缺少prefix_list");
}
List<string> prefixList = System.Text.Json.JsonSerializer.Deserialize<List<string>>(prefixListJson) ?? throw new Exception(); // 不预测前缀
#endregion
#region 加载原型向量
var allLines = File.ReadLines(@".\GameTask\GetGridIcons\训练集原型特征.csv").Skip(1); // 跳过首行列名
Dictionary<string, float[]> prototypes = new Dictionary<string, float[]>();
prototypes = new Dictionary<string, float[]>();
foreach (string line in allLines)
{
var columns = line.Split(",").ToArray();
@@ -65,6 +74,33 @@ public class GridIconsAccuracyTestTask : ISoloTask
prototypes.Add(columns[0], flatData);
}
#endregion
return session;
}
public async Task Start(CancellationToken ct)
{
this.ct = ct;
switch (this.gridScreenName)
{
case GridScreenName.Weapons:
case GridScreenName.Artifacts:
case GridScreenName.CharacterDevelopmentItems:
case GridScreenName.Food:
case GridScreenName.Materials:
case GridScreenName.Gadget:
case GridScreenName.Quest:
case GridScreenName.PreciousItems:
case GridScreenName.Furnishings:
await new ReturnMainUiTask().Start(ct);
await AutoArtifactSalvageTask.OpenBag(this.gridScreenName, this.input, this.logger, this.ct);
break;
default:
logger.LogInformation("{name}暂不支持自动打开，请提前手动打开界面", gridScreenName.GetDescription());
break;
}
using InferenceSession session = LoadModel(out Dictionary<string, float[]> prototypes);
using var ra0 = CaptureToRectArea();
GridScreenParams gridParams = GridScreenParams.Templates[this.gridScreenName];
@@ -89,6 +125,8 @@ public class GridIconsAccuracyTestTask : ISoloTask
await Task.WhenAll(task1, task2);
(string, int) result = task2.Result;
string predName = result.Item1;
int predStarNum = result.Item2;
// 用CV方法得到的结果
using var ra1 = CaptureToRectArea();
@@ -101,14 +139,14 @@ public class GridIconsAccuracyTestTask : ISoloTask
// 统计结果
total_count++;
if (itemName.Contains(result.Item1) && result.Item2 == itemStarNum)
if (itemName.Contains(predName) && predStarNum == itemStarNum)
{
total_acc++;
logger.LogInformation($"{result.Item1}|{result.Item2}星，✔，正确率{total_acc / total_count:0.00}");
logger.LogInformation($"{predName}|{predStarNum}星，✔，正确率{total_acc / total_count:0.00}");
}
else
{
logger.LogInformation($"{result.Item1}|{result.Item2}星，应为：{itemName}|{itemStarNum}星，❌，正确率{total_acc / total_count:0.00}");
logger.LogInformation($"{predName}|{predStarNum}星，应为：{itemName}|{itemStarNum}星，❌，正确率{total_acc / total_count:0.00}");
}
count--;