diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdiparams.info b/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdiparams.info deleted file mode 100644 index 082c148e..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdiparams.info and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdmodel b/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdmodel deleted file mode 100644 index 223b8614..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdmodel and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdiparams b/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/slim_model.onnx similarity index 95% rename from BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdiparams rename to BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/slim_model.onnx index 089594ae..96242d46 100644 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/inference.pdiparams and b/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_det/slim_model.onnx differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdiparams.info b/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdiparams.info deleted file mode 100644 index 923329f5..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdiparams.info and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdmodel b/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdmodel deleted file mode 100644 index dccddcc7..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdmodel and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdiparams b/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/slim_model.onnx similarity index 88% rename from BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdiparams rename to BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/slim_model.onnx index 4c3d9e9c..606b4814 100644 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/inference.pdiparams and b/BetterGenshinImpact/Assets/Model/PaddleOCR/ch_PP-OCRv4_rec/slim_model.onnx differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdiparams.info b/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdiparams.info deleted file mode 100644 index 1cdccfce..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdiparams.info and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdmodel b/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdmodel deleted file mode 100644 index 95acc925..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdmodel and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdiparams b/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/slim_model.onnx similarity index 66% rename from BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdiparams rename to BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/slim_model.onnx index 1b1ea1a9..0b8052b4 100644 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/inference.pdiparams and b/BetterGenshinImpact/Assets/Model/PaddleOCR/chinese_cht_PP-OCRv3_rec_infer/slim_model.onnx differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdiparams b/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdiparams deleted file mode 100644 index 2efedca1..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdiparams and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdiparams.info b/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdiparams.info deleted file mode 100644 index 622d87ba..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdiparams.info and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdmodel b/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdmodel deleted file mode 100644 index 0a6bf1e4..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/inference.pdmodel and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/slim_model.onnx b/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/slim_model.onnx new file mode 100644 index 00000000..5d8910b0 Binary files /dev/null and b/BetterGenshinImpact/Assets/Model/PaddleOCR/en_PP-OCRv3_det_infer/slim_model.onnx differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdiparams.info b/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdiparams.info deleted file mode 100644 index 1cdccfce..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdiparams.info and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdmodel b/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdmodel deleted file mode 100644 index 144978c1..00000000 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdmodel and /dev/null differ diff --git a/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdiparams b/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/slim_model.onnx similarity index 58% rename from BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdiparams rename to BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/slim_model.onnx index 6a7305b7..1d31b0b8 100644 Binary files a/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/inference.pdiparams and b/BetterGenshinImpact/Assets/Model/PaddleOCR/latin_PP-OCRv3_rec_infer/slim_model.onnx differ diff --git a/BetterGenshinImpact/BetterGenshinImpact.csproj b/BetterGenshinImpact/BetterGenshinImpact.csproj index 4bd41176..011d09e0 100644 --- a/BetterGenshinImpact/BetterGenshinImpact.csproj +++ b/BetterGenshinImpact/BetterGenshinImpact.csproj @@ -41,47 +41,47 @@ - + - - - + + + - - - - - - + + + + + + + + - - - + + + - - - + - - - - - - - - - + + + + + + + + + @@ -170,6 +170,11 @@ Wpf Designer + + MSBuild:Compile + Wpf + Designer + diff --git a/BetterGenshinImpact/Core/Config/AllConfig.cs b/BetterGenshinImpact/Core/Config/AllConfig.cs index 6d1b954d..cf4105fe 100644 --- a/BetterGenshinImpact/Core/Config/AllConfig.cs +++ b/BetterGenshinImpact/Core/Config/AllConfig.cs @@ -66,11 +66,11 @@ public partial class AllConfig : ObservableObject [ObservableProperty] private bool _autoFixWin11BitBlt = true; - /// - /// 推理使用的设备 - /// - [ObservableProperty] - private string _inferenceDevice = "CPU"; + // /// + // /// 推理使用的设备 + // /// + // [ObservableProperty] + // private string _inferenceDevice = "CPU"; [ObservableProperty] private List> _nextScheduledTask = []; @@ -135,7 +135,7 @@ public partial class AllConfig : ObservableObject /// 自动战斗配置 /// public AutoFightConfig AutoFightConfig { get; set; } = new(); - + /// /// 自动乐曲配置 - 千音雅集 /// @@ -182,20 +182,28 @@ public partial class AllConfig : ObservableObject /// 原神按键绑定配置 /// public KeyBindingsConfig KeyBindingsConfig { get; set; } = new(); + /// /// 其他配置 /// public OtherConfig OtherConfig { get; set; } = new(); + /// /// 传送相关配置 /// public TpConfig TpConfig { get; set; } = new(); - + /// /// 开发者配置 /// public DevConfig DevConfig { get; set; } = new(); - + + + /// + /// 硬件加速设置 + /// + public HardwareAccelerationConfig HardwareAccelerationConfig { get; set; } = new(); + [JsonIgnore] public Action? OnAnyChangedAction { get; set; } @@ -224,6 +232,7 @@ public partial class AllConfig : ObservableObject ScriptConfig.PropertyChanged += OnAnyPropertyChanged; PathingConditionConfig.PropertyChanged += OnAnyPropertyChanged; DevConfig.PropertyChanged += OnAnyPropertyChanged; + HardwareAccelerationConfig.PropertyChanged += OnAnyPropertyChanged; } public void OnAnyPropertyChanged(object? sender, EventArgs args) diff --git a/BetterGenshinImpact/Core/Config/HardwareAccelerationConfig.cs b/BetterGenshinImpact/Core/Config/HardwareAccelerationConfig.cs new file mode 100644 index 00000000..5dcae9db --- /dev/null +++ b/BetterGenshinImpact/Core/Config/HardwareAccelerationConfig.cs @@ -0,0 +1,75 @@ +using System; +using BetterGenshinImpact.Core.Recognition.ONNX; +using CommunityToolkit.Mvvm.ComponentModel; + +namespace BetterGenshinImpact.Core.Config; + +[Serializable] +public partial class HardwareAccelerationConfig : ObservableObject +{ + /// + /// 推理使用的设备。默认CPU + /// + [ObservableProperty] + private InferenceDeviceType _inferenceDevice = InferenceDeviceType.Cpu; + + /// + /// 是否强制OCR使用CPU推理。在某些环境上使用GPU进行OCR推理会导致性能下降(比如很多使用DirectML推理的情况下)。默认开。 + /// + [ObservableProperty] + private bool _cpuOcr = true; + + #region 一般GPU加速设置 + + /// + /// 强制指定gpu设备,默认为0(使用默认设备) + /// + [ObservableProperty] + private int _gpuDevice = 0; + + /// + /// 附加path，用;分割。默认为空。 + /// + [ObservableProperty] + private string _additionalPath = ""; + + /// + /// 是否输出优化后的模型文件到缓存。注意:在不支持的执行器上使用会导致异常。默认关闭。 + /// + [ObservableProperty] + private bool _optimizedModel = false; + + #endregion + + #region cuda设置 + + /// + /// 强制指定cuda设备,默认为0(使用默认设备) + /// + [ObservableProperty] + private int _cudaDevice = 0; + + /// + /// 自动附加cuda的path。一般情况下用这个就足够了。默认开启。 + /// + [ObservableProperty] + private bool _autoAppendCudaPath = true; + + #endregion + + #region TensorRT缓存设置 + + /// + /// 启用TensorRT缓存。默认开启。不开的话使用TensorRT每次加载模型会卡爆。 + /// + [ObservableProperty] + private bool _enableTensorRtCache = true; + + /// + /// 嵌入式引擎缓存。将引擎缓存嵌入到模型中。默认开启。关闭它可能会提高性能(如果不爆炸的话)。 + /// + [ObservableProperty] + private bool _embedTensorRtCache = true; + + #endregion +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/IOcrService.cs b/BetterGenshinImpact/Core/Recognition/OCR/IOcrService.cs index f30e0d35..9d9f938e 100644 --- a/BetterGenshinImpact/Core/Recognition/OCR/IOcrService.cs +++ b/BetterGenshinImpact/Core/Recognition/OCR/IOcrService.cs @@ -1,5 +1,4 @@ using OpenCvSharp; -using Sdcb.PaddleOCR; namespace BetterGenshinImpact.Core.Recognition.OCR; @@ -9,5 +8,5 @@ public interface IOcrService public string OcrWithoutDetector(Mat mat); - public PaddleOcrResult OcrResult(Mat mat); -} + public OcrResult OcrResult(Mat mat); +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/OcrFactory.cs b/BetterGenshinImpact/Core/Recognition/OCR/OcrFactory.cs index 0338a2bc..dc82f453 100644 --- a/BetterGenshinImpact/Core/Recognition/OCR/OcrFactory.cs +++ b/BetterGenshinImpact/Core/Recognition/OCR/OcrFactory.cs @@ -1,6 +1,4 @@ using System; -using System.Diagnostics; -using System.Threading; using System.Threading.Tasks; namespace BetterGenshinImpact.Core.Recognition.OCR; @@ -15,7 +13,7 @@ public class OcrFactory return type switch { OcrEngineTypes.Paddle => new PaddleOcrService(cultureInfoName), - _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null) }; } @@ -30,4 +28,4 @@ public class OcrFactory } }); } -} +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/OcrResult.cs b/BetterGenshinImpact/Core/Recognition/OCR/OcrResult.cs new file mode 100644 index 00000000..31a858c9 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/OcrResult.cs @@ -0,0 +1,68 @@ +using System.Linq; +using OpenCvSharp; + +namespace BetterGenshinImpact.Core.Recognition.OCR; + +/// +/// Represents a region detected in an OCR result using Paddle OCR. +/// Sdcb.PaddleOCR +/// +public record struct OcrResultRegion(RotatedRect Rect, string Text, float Score); + +public readonly record struct OcrRecognizerResult +{ + /// + /// Initializes a new instance of the struct. + /// + /// The recognized text from the image. + /// The confidence score of the text recognition. + public OcrRecognizerResult(string text, float score) + { + Text = text; + Score = score; + } + + /// + /// The recognized text from the image. + /// + public string Text { get; init; } + + /// + /// The confidence score of the text recognition. + /// + public float Score { get; init; } +} + +/// +/// Represents the OCR result of a paddle object detection model. +/// +public record OcrResult +{ + /// + /// Initializes a new instance of the class with the specified . + /// + /// An array of objects representing the detected text regions. + public OcrResult(OcrResultRegion[] Regions) + { + this.Regions = Regions; + } + + /// + /// Gets an array of objects representing the detected text regions. + /// + /// An array of objects representing the detected text regions. + public OcrResultRegion[] Regions { get; } + + /// + /// Concatenates the text from each object in + /// and returns the resulting string, ordered by the region's center positions. + /// + /// + /// A string containing the concatenated text from each object + /// in , ordered by the region's center positions. + /// + public string Text => string.Join("\n", Regions + .OrderBy(x => x.Rect.Center.Y) + .ThenBy(x => x.Rect.Center.X) + .Select(x => x.Text)); +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/OcrResultExtension.cs b/BetterGenshinImpact/Core/Recognition/OCR/OcrResultExtension.cs new file mode 100644 index 00000000..8ea169da --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/OcrResultExtension.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using BetterGenshinImpact.GameTask.AutoSkip.Model; +using BetterGenshinImpact.View.Drawable; +using OpenCvSharp; + +namespace BetterGenshinImpact.Core.Recognition.OCR; + +public static class OcrResultExtension +{ + public static bool RegionHasText(this OcrResult result, ReadOnlySpan text) + { + foreach (ref readonly var item in result.Regions.AsSpan()) + if (item.Text.AsSpan().Contains(text, StringComparison.InvariantCulture)) + return true; + + return false; + } + + public static OcrResultRegion FindRegionByText(this OcrResult result, ReadOnlySpan text) + { + foreach (ref readonly var item in result.Regions.AsSpan()) + if (item.Text.AsSpan().Contains(text, StringComparison.InvariantCulture)) + return item; + + return default; + } + + public static Rect FindRectByText(this OcrResult result, string text) + { + foreach (ref var item in result.Regions.AsSpan()) + if (item.Text.Contains(text)) + return item.Rect.BoundingRect(); + + return default; + } + + public static List ToRectDrawableList(this OcrResult result, Pen? pen = null) + { + return result.Regions.Select(item => item.Rect.BoundingRect().ToRectDrawable(pen)).ToList(); + } + + public static List ToRectDrawableListOffset(this OcrResult result, int offsetX, int offsetY, + Pen? pen = null) + { + return result.Regions.Select(item => item.Rect.BoundingRect().ToRectDrawable(offsetX, offsetY, pen)).ToList(); + } + + public static PaddleOcrResultRect ToOcrResultRect(this OcrResultRegion region) + { + return new PaddleOcrResultRect(region.Rect.BoundingRect(), region.Text, region.Score); + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/PaddleOcrResultExtension.cs b/BetterGenshinImpact/Core/Recognition/OCR/PaddleOcrResultExtension.cs deleted file mode 100644 index 794c8062..00000000 --- a/BetterGenshinImpact/Core/Recognition/OCR/PaddleOcrResultExtension.cs +++ /dev/null @@ -1,67 +0,0 @@ -using BetterGenshinImpact.GameTask.AutoSkip.Model; -using BetterGenshinImpact.View.Drawable; -using OpenCvSharp; -using Sdcb.PaddleOCR; -using System; -using System.Collections.Generic; -using System.Drawing; -using System.Linq; - -namespace BetterGenshinImpact.Core.Recognition.OCR; - -public static class PaddleOcrResultExtension -{ - public static bool RegionHasText(this PaddleOcrResult result, ReadOnlySpan text) - { - foreach (ref readonly PaddleOcrResultRegion item in result.Regions.AsSpan()) - { - if (item.Text.AsSpan().Contains(text, StringComparison.InvariantCulture)) - { - return true; - } - } - - return false; - } - - public static PaddleOcrResultRegion FindRegionByText(this PaddleOcrResult result, ReadOnlySpan text) - { - foreach (ref readonly PaddleOcrResultRegion item in result.Regions.AsSpan()) - { - if (item.Text.AsSpan().Contains(text, StringComparison.InvariantCulture)) - { - return item; - } - } - - return default; - } - - public static Rect FindRectByText(this PaddleOcrResult result, string text) - { - foreach (ref PaddleOcrResultRegion item in result.Regions.AsSpan()) - { - if (item.Text.Contains(text)) - { - return item.Rect.BoundingRect(); - } - } - - return default; - } - - public static List ToRectDrawableList(this PaddleOcrResult result, Pen? pen = null) - { - return result.Regions.Select(item => item.Rect.BoundingRect().ToRectDrawable(pen)).ToList(); - } - - public static List ToRectDrawableListOffset(this PaddleOcrResult result, int offsetX, int offsetY, Pen? pen = null) - { - return result.Regions.Select(item => item.Rect.BoundingRect().ToRectDrawable(offsetX, offsetY, pen)).ToList(); - } - - public static PaddleOcrResultRect ToOcrResultRect(this PaddleOcrResultRegion region) - { - return new PaddleOcrResultRect(region.Rect.BoundingRect(), region.Text, region.Score); - } -} diff --git a/BetterGenshinImpact/Core/Recognition/OCR/PaddleOcrService.cs b/BetterGenshinImpact/Core/Recognition/OCR/PaddleOcrService.cs deleted file mode 100644 index 2dc7a648..00000000 --- a/BetterGenshinImpact/Core/Recognition/OCR/PaddleOcrService.cs +++ /dev/null @@ -1,102 +0,0 @@ -using BetterGenshinImpact.Core.Config; -using OpenCvSharp; -using Sdcb.PaddleInference; -using Sdcb.PaddleOCR; -using Sdcb.PaddleOCR.Models; -using System; -using System.Diagnostics; -using System.IO; -using static Vanara.PInvoke.Gdi32; - -namespace BetterGenshinImpact.Core.Recognition.OCR; - -public class PaddleOcrService : IOcrService -{ - private static readonly object locker = new(); - - /// - /// Usage: - /// https://github.com/sdcb/PaddleSharp/blob/master/docs/ocr.md - /// 模型列表: - /// https://github.com/PaddlePaddle/PaddleOCR/blob/release/2.5/doc/doc_ch/models_list.md - /// - private readonly PaddleOcrAll _paddleOcrAll; - - public PaddleOcrService(string? cultureInfoName = null) - { - var path = Global.Absolute(@"Assets\Model\PaddleOcr"); - DetectionModel localDetModel; - RecognizationModel localRecModel; - FullOcrModel model; - switch (cultureInfoName) - { - case "zh-Hant": - localDetModel = DetectionModel.FromDirectory(Path.Combine(path, "ch_PP-OCRv4_det"), ModelVersion.V4); // 和简中共用一下det - localRecModel = RecognizationModel.FromDirectory(Path.Combine(path, "chinese_cht_PP-OCRv3_rec_infer"), Path.Combine(path, "chinese_cht_dict.txt"), ModelVersion.V3); - //model = OnlineFullModels.TraditionalChineseV3.DownloadAsync().Result; - break; - case "fr": - localDetModel = DetectionModel.FromDirectory(Path.Combine(path, "en_PP-OCRv3_det_infer"), ModelVersion.V3); - localRecModel = RecognizationModel.FromDirectory(Path.Combine(path, "latin_PP-OCRv3_rec_infer"), Path.Combine(path, "latin_dict.txt"), ModelVersion.V3); - break; - default: - localDetModel = DetectionModel.FromDirectory(Path.Combine(path, "ch_PP-OCRv4_det"), ModelVersion.V4); - localRecModel = RecognizationModel.FromDirectory(Path.Combine(path, "ch_PP-OCRv4_rec"), Path.Combine(path, "ppocr_keys_v1.txt"), ModelVersion.V4); - - break; - } - model = new FullOcrModel(localDetModel, localRecModel); - // Action device = TaskContext.Instance().Config.InferenceDevice switch - // { - // "CPU" => PaddleDevice.Onnx(), - // "GPU_DirectML" => PaddleDevice.Onnx(), - // _ => throw new InvalidEnumArgumentException("无效的推理设备") - // }; - _paddleOcrAll = new PaddleOcrAll(model, PaddleDevice.Onnx()) - { - AllowRotateDetection = false, /* 允许识别有角度的文字 */ - Enable180Classification = false /* 允许识别旋转角度大于90度的文字 */ - }; - - // System.AccessViolationException - // https://github.com/babalae/better-genshin-impact/releases/latest - // 下载并解压到相同目录下 - } - - public string Ocr(Mat mat) - { - return OcrResult(mat).Text; - } - - public PaddleOcrResult OcrResult(Mat mat) - { - if (mat.Channels() == 4) - { - using var mat3 = mat.CvtColor(ColorConversionCodes.BGRA2BGR); - return _OcrResult(mat3); - } - return _OcrResult(mat); - } - - private PaddleOcrResult _OcrResult(Mat mat) - { - lock (locker) - { - long startTime = Stopwatch.GetTimestamp(); - var result = _paddleOcrAll.Run(mat); - TimeSpan time = Stopwatch.GetElapsedTime(startTime); - Debug.WriteLine($"PaddleOcr 耗时 {time.TotalMilliseconds}ms 结果: {result.Text}"); - return result; - } - } - - public string OcrWithoutDetector(Mat mat) - { - lock (locker) - { - var str = _paddleOcrAll.Recognizer.Run(mat).Text; - Debug.WriteLine($"PaddleOcrWithoutDetector 结果: {str}"); - return str; - } - } -} diff --git a/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrOperationImpl.cs b/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrOperationImpl.cs new file mode 100644 index 00000000..80217718 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrOperationImpl.cs @@ -0,0 +1,41 @@ +using OpenCvSharp; + +namespace BetterGenshinImpact.Core.Recognition.OCR.engine; + +/// +/// 实现PPOCR的自定义操作，代码翻自python +/// +public class OcrOperationImpl +{ + /// + /// 不支持 chw 之类的顺序 + /// https://github.com/PaddlePaddle/PaddleOCR/blob/0ee4094988c568077bba35ddb239030ced1ff270/ppocr/data/imaug/operators.py#L62 + /// + public static Mat NormalizeImageOperation(Mat data, + float? scale, // scale float32 + float[]? mean, //mean + float[]? std //std + ) + { + scale ??= 0.00392156862745f; + mean ??= [0.485f, 0.456f, 0.406f]; + std ??= [0.229f, 0.224f, 0.225f]; + var result = new Mat(); + data.ConvertTo(result, MatType.CV_32FC3, (double)scale); + Mat[] bgr = []; + try + { + bgr = result.Split(); + for (var i = 0; i < bgr.Length; ++i) + bgr[i].ConvertTo(bgr[i], MatType.CV_32FC1, 1 / std[i], (0.0 - mean[i]) / std[i]); + + Cv2.Merge(bgr, result); + } + finally + { + foreach (var channel in bgr) channel.Dispose(); + } + + return result; + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrUtils.cs b/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrUtils.cs new file mode 100644 index 00000000..f8271b68 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrUtils.cs @@ -0,0 +1,180 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq; +using BetterGenshinImpact.Core.Recognition.OCR.engine; +using BetterGenshinImpact.Core.Recognition.OCR.paddle.data; +using BetterGenshinImpact.Core.Recognition.OpenCv; +using Microsoft.ML.OnnxRuntime.Tensors; +using OpenCvSharp; +using OpenCvSharp.Dnn; + +namespace BetterGenshinImpact.Core.Recognition.OCR; + +public static class OcrUtils +{ + /// + /// 预处理速度比unsafe快5倍以上,且吃的资源还少 + /// + /// 输入图像，若不是灰度图会转换 + /// tensor的Memory，用完需要释放 + /// + public static Tensor ToTensorYapDnn(Mat inputImage, out IMemoryOwner tensorMemoryOwnser) + { + using var rt = new ResourcesTracker(); + Mat dst; + // 221*32是个什么鬼 + if (inputImage.Channels() > 1) + { + var resize = rt.T(ResizeHelper.ResizeTo(inputImage, 221, 32)); + dst = rt.NewMat(resize.Size(), MatType.CV_8UC1, Scalar.Black); + Cv2.CvtColor(resize, dst, ColorConversionCodes.BGR2GRAY); + } + else + { + dst = rt.T(ResizeHelper.ResizeTo(inputImage, 221, 32)); + } + + // 填充到 384x32 + var padded = rt.NewMat(new Size(384, 32), MatType.CV_8UC1, Scalar.Black); + padded[new Rect(0, 0, 221, 32)] = dst; + // 使用向量运算代替循环 + var blob = rt.T(CvDnn.BlobFromImage(padded, 1.0 / 255.0, default, default, false, false)); + var nCols = padded.Cols * padded.Rows; + tensorMemoryOwnser = MemoryPool.Shared.Rent(nCols); + // 内存复制，如果直接传指针构建的话速度还不如多复制一份 + blob.AsSpan().CopyTo(tensorMemoryOwnser.Memory.Span); + return new DenseTensor(tensorMemoryOwnser.Memory[..nCols], [1, 1, 32, 384]); + } + + /// + /// 用于Det模型 + /// 归一化,标准化并返回Tensor。 + ///
+ /// 归一化:固定范围归一化 + ///
+ /// 标准化: + /// Z-Score Normalization + /// + public static Tensor NormalizeToTensorDnn(Mat src, + float? scale, // scale float32 + float[]? mean, //mean + float[]? std, //std + out IMemoryOwner tensorMemoryOwner, bool swapRb = false, bool crop = false, Size size = default) + + { + using var rt = new ResourcesTracker(); + // 获取图像参数 + var channels = src.Channels(); + if (channels != 3) + throw new ArgumentException($"图像通道数必须为3,当前为{channels}"); + var data = rt.T(OcrOperationImpl.NormalizeImageOperation(src, scale, mean, std)); + // 使用DNN模块创建blob + var blob = rt.T(CvDnn.BlobFromImage( + data, + 1.0, + size, + default, + swapRb, + crop + )); + // 租用内存并复制数据 + var total = blob.Total(); + tensorMemoryOwner = MemoryPool.Shared.Rent((int)total); + blob.AsSpan().CopyTo(tensorMemoryOwner.Memory.Span); + // 计算输出形状 + return new DenseTensor( + tensorMemoryOwner.Memory[..(int)total], + new[] { 1, channels, data.Rows, data.Cols } + ); + } + + /// + /// 不支持通道转换 + ///
+ /// 用于PP-OCR的Rec模型，调整大小之后再归一化到-1~1，之后转换为Tensor + /// + public static Tensor resize_norm_img(Mat img, OcrShape image_shape, + out IMemoryOwner tensorMemoryOwner, bool padding = true, + InterpolationFlags interpolation = InterpolationFlags.Linear) + { + using var rt = new ResourcesTracker(); + var imgC = image_shape.Channel; + var imgH = image_shape.Height; + var imgW = image_shape.Width; + + var h = img.Height; + var w = img.Width; + + int resized_w; + + var resizedImage = rt.NewMat(); + if (!padding) + { + Cv2.Resize(img, resizedImage, new Size(imgW, imgH), 0, 0, interpolation); + // resized_w = imgW; + } + else + { + var ratio = w / (double)h; + resized_w = Math.Ceiling(imgH * ratio) > imgW ? imgW : (int)Math.Ceiling(imgH * ratio); + Cv2.Resize(img, resizedImage, new Size(resized_w, imgH), 0, 0, interpolation); + } + + /* + resized_image / 255 + resized_image -= 0.5 + resized_image /= 0.5 + */ + // 归一化 + // resizedImage.ConvertTo(resizedImage, MatType.CV_32F, 2 / 255f, 1); + var blob = rt.T(CvDnn.BlobFromImage( + resizedImage, + 2 / 255f, + default, + new Scalar(127.5, 127.5, 127.5), + false, + false + )); + + var total = blob.Total(); + tensorMemoryOwner = MemoryPool.Shared.Rent((int)total); + blob.AsSpan().CopyTo(tensorMemoryOwner.Memory.Span); + return new DenseTensor( + tensorMemoryOwner.Memory[..(int)total], + new[] { 1, resizedImage.Channels(), resizedImage.Rows, resizedImage.Cols } + ); + } + + /// + /// Gets a label by its index. + /// + /// The index of the label. + /// The labels to search for the index. + /// The label at the specified index. + public static string GetLabelByIndex(int i, IReadOnlyList labels) + { + return i switch + { + var x when x > 0 && x <= labels.Count => labels[x - 1], + var x when x == labels.Count + 1 => " ", + _ => throw new Exception( + $"Unable to GetLabelByIndex: index {i} out of range {labels.Count}, OCR model or labels not matched?") + }; + } + + public static Mat Tensor2Mat(Tensor tensor) + { + var dimensions = tensor.Dimensions; + if (dimensions.Length !=4 || dimensions[0] != 1 || dimensions[1] != 1) + { + throw new ArgumentException($"wrong tensor shape: {string.Join(",", dimensions.ToArray())}"); + } + if (tensor is not DenseTensor denseTensor) + return Mat.FromPixelData(dimensions[2], dimensions[3], MatType.CV_32FC1, tensor.ToArray()); + var mat = new Mat(new Size(dimensions[3], dimensions[2]), MatType.CV_32FC1); + denseTensor.Buffer.Span.CopyTo(mat.AsSpan()); + return mat; + + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrVersionConfig.cs b/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrVersionConfig.cs new file mode 100644 index 00000000..d0c6c58d --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/engine/OcrVersionConfig.cs @@ -0,0 +1,37 @@ +using BetterGenshinImpact.Core.Recognition.OCR.paddle.data; + +namespace BetterGenshinImpact.Core.Recognition.OCR; + +/// +/// ppocr的版本配置 +/// +public readonly record struct OcrVersionConfig( + string Name, + OcrImgMode Mode, + bool ChannelFirst, + OcrNormalizeImage NormalizeImage, + OcrShape Shape) +{ + // 参数来自 https://github.com/PaddlePaddle/PaddleOCR/tree/main/configs/det/PP-OCRv3 + + public static OcrVersionConfig PpOcrV3 = new( + "PP-OCRv3", + OcrImgMode.BGR, + false, + new OcrNormalizeImage( + 1.0f / 255.0f, + [0.485f, 0.456f, 0.406f], + [0.229f, 0.224f, 0.225f] + ), new OcrShape(3, 320, 48) + ); + + public static OcrVersionConfig PpOcrV4 = new( + "PP-OCRv4", + OcrImgMode.BGR, + false, + new OcrNormalizeImage( + 1.0f / 255.0f, + [0.485f, 0.456f, 0.406f], + [0.229f, 0.224f, 0.225f] + ), new OcrShape(3, 320, 48)); +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrImgMode.cs b/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrImgMode.cs new file mode 100644 index 00000000..7d734627 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrImgMode.cs @@ -0,0 +1,9 @@ +namespace BetterGenshinImpact.Core.Recognition.OCR; +/// +/// 图像的颜色顺序 +/// +public enum OcrImgMode +{ + BGR, + RGB +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrMatOrder.cs b/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrMatOrder.cs new file mode 100644 index 00000000..28f5a899 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrMatOrder.cs @@ -0,0 +1,11 @@ +namespace BetterGenshinImpact.Core.Recognition.OCR; +/// +/// Mat的通道顺序 +/// hwc: height width channel +/// chw: channel height width +/// +public enum OcrMatOrder +{ + Hwc, + Chw +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrNormalizeImage.cs b/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrNormalizeImage.cs new file mode 100644 index 00000000..b5077403 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrNormalizeImage.cs @@ -0,0 +1,5 @@ +namespace BetterGenshinImpact.Core.Recognition.OCR; +/// +/// 标准归一化的三个参数 +/// +public record OcrNormalizeImage(float Scale, float[] Mean, float[] Std); \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrShape.cs b/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrShape.cs new file mode 100644 index 00000000..82bfe847 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/engine/data/OcrShape.cs @@ -0,0 +1,6 @@ +namespace BetterGenshinImpact.Core.Recognition.OCR.paddle.data; + +/// +/// 图像形状表示 +/// +public readonly record struct OcrShape(int Channel, int Width, int Height); \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/paddle/Det.cs b/BetterGenshinImpact/Core/Recognition/OCR/paddle/Det.cs new file mode 100644 index 00000000..e41af092 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/paddle/Det.cs @@ -0,0 +1,184 @@ +using System; +using System.Linq; +using BetterGenshinImpact.Core.Recognition.ONNX; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using OpenCvSharp; + +namespace BetterGenshinImpact.Core.Recognition.OCR.paddle; + +public class Det +{ + private readonly OcrVersionConfig _config; + private readonly InferenceSession _session; + + public Det(BgiOnnxModel model, OcrVersionConfig config) + { + _config = config; + _session = BgiOnnxFactory.Instance.CreateInferenceSession(model,true); + } + + /// Gets or sets the maximum size for resizing the input image. + public int? MaxSize { get; set; } = 1536; + + /// Gets or sets the size for dilation during preprocessing. + public int? DilatedSize { get; set; } = 2; + + /// Gets or sets the score threshold for filtering out possible text boxes. + public float? BoxScoreThreshold { get; set; } = 0.7f; + + /// Gets or sets the threshold to binarize the text region. + public float? BoxThreshold { get; set; } = 0.3f; + + /// Gets or sets the minimum size of the text boxes to be considered as valid. + public int MinSize { get; set; } = 3; + + /// Gets or sets the ratio for enlarging text boxes during post-processing. + public float UnclipRatio { get; set; } = 2.0f; + + ~Det() + { + _session.Dispose(); + } + + public RotatedRect[] Run(Mat src) + { + using var pred = RunRaw(src, out var resizedSize); + using Mat cbuf = new(); + //OpenCvSharp.OpenCVException: 0 <= _colRange.start && _colRange.start <= _colRange.end && _colRange.end <= m.cols + using var roi = pred[0, resizedSize.Height, 0, resizedSize.Width]; + roi.ConvertTo(cbuf, MatType.CV_8UC1, 255); + using Mat dilated = new(); + using var binary = BoxThreshold != null + ? cbuf.Threshold((int)(BoxThreshold * 255), 255, ThresholdTypes.Binary) + : cbuf; + if (DilatedSize != null) + { + using var ones = + Cv2.GetStructuringElement(MorphShapes.Rect, new Size(DilatedSize.Value, DilatedSize.Value)); + Cv2.Dilate(binary, dilated, ones); + } + else + { + Cv2.CopyTo(binary, dilated); + } + + var contours = dilated.FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxSimple); + // var size = src.Size(); + var scaleRate = 1.0 * src.Width / resizedSize.Width; + + var rects = contours + .Where(x => BoxScoreThreshold == null || GetScore(x, pred) > BoxScoreThreshold) + .Select(Cv2.MinAreaRect) + .Where(x => x.Size.Width > MinSize && x.Size.Height > MinSize) + .Select(rect => + { + var minEdge = Math.Min(rect.Size.Width, rect.Size.Height); + Size2f newSize = new( + (rect.Size.Width + UnclipRatio * minEdge) * scaleRate, + (rect.Size.Height + UnclipRatio * minEdge) * scaleRate); + RotatedRect largerRect = new(rect.Center * scaleRate, newSize, rect.Angle); + return largerRect; + }) + .OrderBy(v => v.Center.Y) + .ThenBy(v => v.Center.X) + .ToArray(); + //{ + // using Mat demo = dilated.CvtColor(ColorConversionCodes.GRAY2RGB); + // demo.DrawContours(contours, -1, Scalar.Red); + // Image(demo).Dump(); + //} + return rects; + } + + public Mat RunRaw(Mat src, out Size resizedSize) + { + var padded = src.Channels() switch + { + 4 => src.CvtColor(ColorConversionCodes.BGRA2BGR), + 1 => src.CvtColor(ColorConversionCodes.GRAY2BGR), + 3 => src, + var x => throw new Exception($"Unexpect src channel: {x}, allow: (1/3/4)") + }; + using (var resized = MatResize(padded, MaxSize)) + { + resizedSize = new Size(resized.Width, resized.Height); + padded = MatPadding32(resized); + } + + using (var _ = padded) + { + var inputTensor = OcrUtils.NormalizeToTensorDnn(padded, _config.NormalizeImage.Scale, + _config.NormalizeImage.Mean, _config.NormalizeImage.Std, out var owner); + using (owner) + { + lock (_session) + { + using IDisposableReadOnlyCollection results = _session.Run([ + NamedOnnxValue.CreateFromTensor(_session.InputNames[0], inputTensor) + ]); + var output = results[0]; + if (output.ElementType is not TensorElementType.Float) + throw new Exception($"Unexpected output tensor type: {output.ElementType}"); + + if (output.ValueType is not OnnxValueType.ONNX_TYPE_TENSOR) + throw new Exception($"Unexpected output tensor value type: {output.ValueType}"); + var outputTensor = output.AsTensor(); + return OcrUtils.Tensor2Mat(tensor: outputTensor); + // 因为一个已知bug,tensor中内存在dml下使用完后会被释放掉,锁之外的代码会报错 + } + } + } + } + + private static Mat MatPadding32(Mat src) + { + var size = src.Size(); + Size newSize = new( + 32 * Math.Ceiling(1.0 * size.Width / 32), + 32 * Math.Ceiling(1.0 * size.Height / 32)); + return src.CopyMakeBorder(0, newSize.Height - size.Height, 0, newSize.Width - size.Width, BorderTypes.Constant, + Scalar.Black); + } + + private static Mat MatResize(Mat src, int? maxSize) + { + if (maxSize == null) return src.Clone(); + + var size = src.Size(); + var longEdge = Math.Max(size.Width, size.Height); + var scaleRate = 1.0 * maxSize.Value / longEdge; + return scaleRate < 1.0 ? src.Resize(default, scaleRate, scaleRate) : src.Clone(); + } + + private static float GetScore(Point[] contour, Mat pred) + { + var width = pred.Width; + var height = pred.Height; + var boxX = contour.Select(v => v.X).ToArray(); + var boxY = contour.Select(v => v.Y).ToArray(); + + var xmin = Math.Clamp(boxX.Min(), 0, width - 1); + var xmax = Math.Clamp(boxX.Max(), 0, width - 1); + var ymin = Math.Clamp(boxY.Min(), 0, height - 1); + var ymax = Math.Clamp(boxY.Max(), 0, height - 1); + + var rootPoints = contour + .Select(v => new Point(v.X - xmin, v.Y - ymin)) + .ToArray(); + using Mat mask = new(ymax - ymin + 1, xmax - xmin + 1, MatType.CV_8UC1, Scalar.Black); + mask.FillPoly(new[] { rootPoints }, new Scalar(1)); + + using var croppedMat = pred[ymin, ymax + 1, xmin, xmax + 1]; + var score = (float)croppedMat.Mean(mask).Val0; + + // Debug + //{ + // using Mat cu = new Mat(); + // croppedMat.ConvertTo(cu, MatType.CV_8UC1, 255); + // Util.HorizontalRun(true, Image(cu), Image(mask), score).Dump(); + //} + + return score; + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/paddle/PaddleOcrService.cs b/BetterGenshinImpact/Core/Recognition/OCR/paddle/PaddleOcrService.cs new file mode 100644 index 00000000..a208eb56 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/paddle/PaddleOcrService.cs @@ -0,0 +1,130 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using BetterGenshinImpact.Core.Config; +using BetterGenshinImpact.Core.Recognition.OCR.paddle; +using BetterGenshinImpact.Core.Recognition.ONNX; +using OpenCvSharp; + +namespace BetterGenshinImpact.Core.Recognition.OCR; + +public class PaddleOcrService : IOcrService +{ + /// + /// Usage: + /// https://github.com/sdcb/PaddleSharp/blob/master/docs/ocr.md + /// 模型列表: + /// https://github.com/PaddlePaddle/PaddleOCR/blob/release/2.5/doc/doc_ch/models_list.md + /// + private readonly Det localDetModel; + + private readonly Rec localRecModel; + + public PaddleOcrService(string? cultureInfoName = null) + { + var path = Global.Absolute(@"Assets\Model\PaddleOcr"); + + switch (cultureInfoName) + { + case "zh-Hant": + localDetModel = new Det(BgiOnnxModel.PaddleOcrChDet, OcrVersionConfig.PpOcrV4); + localRecModel = new Rec(BgiOnnxModel.PaddleOcrChtRec, Path.Combine(path, "chinese_cht_dict.txt"), + OcrVersionConfig.PpOcrV3); + break; + case "fr": + localDetModel = new Det(BgiOnnxModel.PaddleOcrEnDet, OcrVersionConfig.PpOcrV3); + localRecModel = new Rec(BgiOnnxModel.PaddleOcrLatinRec, Path.Combine(path, "latin_dict.txt"), + OcrVersionConfig.PpOcrV3); + break; + default: + localDetModel = new Det(BgiOnnxModel.PaddleOcrChDet, OcrVersionConfig.PpOcrV4); + localRecModel = new Rec(BgiOnnxModel.PaddleOcrChRec, Path.Combine(path, "ppocr_keys_v1.txt"), + OcrVersionConfig.PpOcrV4); + + break; + } + } + + /// + /// 推荐传入三通道BGR mat，虽然四通道和单通道也做了兼容，但是三通道最快 + /// + public string Ocr(Mat mat) + { + return OcrResult(mat).Text; + } + + /// + /// 推荐传入三通道BGR mat，虽然四通道和单通道也做了兼容，但是三通道最快 + /// + public OcrResult OcrResult(Mat mat) + { + if (mat.Channels() == 4) + { + using var mat3 = mat.CvtColor(ColorConversionCodes.BGRA2BGR); + return _OcrResult(mat3); + } + + return _OcrResult(mat); + } + + /// + /// 推荐传入三通道BGR mat，虽然四通道和单通道也做了兼容，但是三通道最快 + /// + public string OcrWithoutDetector(Mat mat) + { + var str = localRecModel.Run(mat).Text; + Debug.WriteLine($"PaddleOcrWithoutDetector 结果: {str}"); + return str; + } + + private OcrResult _OcrResult(Mat mat) + { + var startTime = Stopwatch.GetTimestamp(); + var result = RunAll(mat); + var time = Stopwatch.GetElapsedTime(startTime); + Debug.WriteLine($"PaddleOcr 耗时 {time.TotalMilliseconds}ms 结果: {result.Text}"); + return result; + } + + /// + /// 推荐传入三通道BGR mat，虽然四通道和单通道也做了兼容，但是三通道最快 + /// + private OcrResult RunAll(Mat src, int recognizeBatchSize = 0) + { + var rects = localDetModel.Run(src); + Mat[] mats = + rects.Select(rect => + { + var roi = src[GetCropedRect(rect.BoundingRect(), src.Size())]; + return roi; + }) + .ToArray(); + try + { + return new OcrResult(localRecModel.Run(mats, recognizeBatchSize) + .Select((result, i) => new OcrResultRegion(rects[i], result.Text, result.Score)) + .ToArray()); + } + finally + { + foreach (var mat in mats) mat.Dispose(); + } + } + + /// + /// Gets the cropped region of the source image specified by the given rectangle, clamping the rectangle coordinates to + /// the image bounds. + /// + /// The rectangle to crop. + /// The size of the source image. + /// The cropped rectangle. + private static Rect GetCropedRect(Rect rect, Size size) + { + return Rect.FromLTRB( + Math.Clamp(rect.Left, 0, size.Width), + Math.Clamp(rect.Top, 0, size.Height), + Math.Clamp(rect.Right, 0, size.Width), + Math.Clamp(rect.Bottom, 0, size.Height)); + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OCR/paddle/Rec.cs b/BetterGenshinImpact/Core/Recognition/OCR/paddle/Rec.cs new file mode 100644 index 00000000..f5fe1cb2 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OCR/paddle/Rec.cs @@ -0,0 +1,177 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using BetterGenshinImpact.Core.Recognition.OCR.paddle.data; +using BetterGenshinImpact.Core.Recognition.ONNX; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using OpenCvSharp; + +namespace BetterGenshinImpact.Core.Recognition.OCR.paddle; + +public class Rec +{ + private readonly OcrVersionConfig _config; + private readonly IReadOnlyList _labels; + private readonly InferenceSession _session; + + public Rec(BgiOnnxModel model, string labelFilePath, OcrVersionConfig config) + { + _config = config; + _session = BgiOnnxFactory.Instance.CreateInferenceSession(model,true); + + + _labels = File.ReadAllLines(labelFilePath); + } + + ~Rec() + { + lock (_session) + { + _session.Dispose(); + } + } + + /// + /// Run OCR recognition on multiple images in batches. + /// + /// Array of images for OCR recognition. + /// Size of the batch to run OCR recognition on. + /// Array of instances corresponding to OCR recognition results of the images. + public OcrRecognizerResult[] Run(Mat[] srcs, int batchSize = 0) + { + if (srcs.Length == 0) return []; + + var chooseBatchSize = batchSize != 0 ? batchSize : Math.Min(8, Environment.ProcessorCount); + + return srcs + .Select((x, i) => (mat: x, i)) + .OrderBy(x => x.mat.Width) + .Chunk(chooseBatchSize) + .Select(x => (result: RunMulti(x.Select(x1 => x1.mat).ToArray()), ids: x.Select(x1 => x1.i).ToArray())) + .SelectMany(x => x.result.Zip(x.ids, (result, i) => (result, i))) + .OrderBy(x => x.i) + .Select(x => x.result) + .ToArray(); + } + + public OcrRecognizerResult Run(Mat src) + { + return RunMulti([src]).Single(); + } + + private OcrRecognizerResult[] RunMulti(Mat[] srcs) + { + if (srcs.Length == 0) return []; + + for (var i = 0; i < srcs.Length; ++i) + { + var src = srcs[i]; + if (src.Empty()) + throw new ArgumentException($"src[{i}] size should not be 0, wrong input picture provided?"); + } + + var modelHeight = _config.Shape.Height; + var maxWidth = (int)Math.Ceiling(srcs.Max(src => + { + var size = src.Size(); + return 1.0 * size.Width / size.Height * modelHeight; + })); + List> owners = []; + (int[], float[])[] resultTensors; + try + { + resultTensors = srcs + .Select(src => + { + using var channel3 = src.Channels() switch + { + 4 => src.CvtColor(ColorConversionCodes.BGRA2BGR), + 1 => src.CvtColor(ColorConversionCodes.GRAY2BGR), + 3 => src, + var x => throw new Exception($"Unexpect src channel: {x}, allow: (1/3/4)") + }; + var result = OcrUtils.resize_norm_img(channel3, new OcrShape(3, maxWidth, modelHeight), + out var owner); + lock (owners) + { + owners.Add(owner); + } + + return result; + }) + .Select( + inputTensor => + { + lock (_session) + { + // 多线程推理会出现问题，加锁解决。 + using IDisposableReadOnlyCollection results = _session.Run([ + NamedOnnxValue.CreateFromTensor(_session.InputNames[0], inputTensor) + ]); + var output = results[0]; + if (output.ElementType is not TensorElementType.Float) + throw new Exception($"Unexpected output tensor type: {output.ElementType}"); + + if (output.ValueType is not OnnxValueType.ONNX_TYPE_TENSOR) + throw new Exception($"Unexpected output tensor value type: {output.ValueType}"); + var tensor = output.AsTensor(); + // 因为一个已知bug,tensor中内存在dml下使用完后会被释放掉,锁之外的代码会报错 + return (tensor.Dimensions.ToArray(), tensor.ToArray()); + } + } + ).ToArray(); + } + finally + { + owners.ForEach(x => { x.Dispose(); }); + } + + return resultTensors.SelectMany( + resultTensor => + { + var resultArray = resultTensor.Item2; + var resultShape = resultTensor.Item1; + GCHandle dataHandle = default; + try + { + dataHandle = GCHandle.Alloc(resultArray, GCHandleType.Pinned); + var dataPtr = dataHandle.AddrOfPinnedObject(); + var labelCount = resultShape[2]; + var charCount = resultShape[1]; + + return Enumerable.Range(0, resultShape[0]) + .Select(i => + { + StringBuilder sb = new(); + var lastIndex = 0; + float score = 0; + for (var n = 0; n < charCount; ++n) + { + using var mat = Mat.FromPixelData(1, labelCount, MatType.CV_32FC1, + dataPtr + (n + i * charCount) * labelCount * sizeof(float)); + var maxIdx = new int[2]; + mat.MinMaxIdx(out _, out var maxVal, [], maxIdx); + + if (maxIdx[1] > 0 && !(n > 0 && maxIdx[1] == lastIndex)) + { + score += (float)maxVal; + sb.Append(OcrUtils.GetLabelByIndex(maxIdx[1], _labels)); + } + lastIndex = maxIdx[1]; + } + return new OcrRecognizerResult(sb.ToString(), score / sb.Length); + }) + .ToArray(); + } + finally + { + dataHandle.Free(); + } + }).ToArray(); + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/BgiOnnxFactory.cs b/BetterGenshinImpact/Core/Recognition/ONNX/BgiOnnxFactory.cs new file mode 100644 index 00000000..b7303554 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/ONNX/BgiOnnxFactory.cs @@ -0,0 +1,473 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using BetterGenshinImpact.Core.Config; +using BetterGenshinImpact.GameTask; +using BetterGenshinImpact.Model; +using Microsoft.Extensions.Logging; +using Microsoft.ML.OnnxRuntime; +using Microsoft.Win32; +using Vanara; + +namespace BetterGenshinImpact.Core.Recognition.ONNX; + +public class BgiOnnxFactory : Singleton +{ + private static readonly ILogger Logger = App.GetLogger(); + + + public ProviderType[] ProviderTypes { get; } + public int DmlDeviceId { get; } + public int CudaDeviceId { get; } + public bool OptimizedModel { get; } + public bool TrtUseEmbedMode { get; } + public bool EnableCache { get; } + public bool CpuOcr { get; } + + /// + /// 缓存模型路径。如果一开始使用缓存就一直使用缓存文件，如果没有使用缓存就一直使用原始模型路径。 + ///
+ /// 这样能避免并发加载模型问题。比如使用了未完全构建好的缓存文件，导致模型加载失败。 + /// + private ConcurrentDictionary _cachedModelPaths = new(); + + + public BgiOnnxFactory() + { + var config = TaskContext.Instance().Config.HardwareAccelerationConfig; + if (config.AutoAppendCudaPath) + { + AppendCudaPath(); + } + + if (string.IsNullOrWhiteSpace(config.AdditionalPath)) + { + AppendPath(config.AdditionalPath.Split(Path.PathSeparator)); + } + + ProviderTypes = GetProviderType(config.InferenceDevice, CudaDeviceId, DmlDeviceId); + OptimizedModel = config.OptimizedModel; + CudaDeviceId = config.CudaDevice; + DmlDeviceId = config.GpuDevice; + TrtUseEmbedMode = config.EmbedTensorRtCache; + EnableCache = config.EnableTensorRtCache; + CpuOcr = config.CpuOcr; + Logger.LogDebug( + "[ONNX]启用的provider:{Device},初始化参数: InferenceDevice={InferenceDevice}, OptimizedModel={OptimizedModel}, CudaDeviceId={CudaDeviceId}, DmlDeviceId={DmlDeviceId}, EmbedTensorRtCache={EmbedTensorRtCache}, EnableTensorRtCache={EnableTensorRtCache}, CpuOcr={CpuOcr}", + string.Join(",", ProviderTypes.Select(Enum.GetName)), + config.InferenceDevice, + OptimizedModel, + CudaDeviceId, + DmlDeviceId, + TrtUseEmbedMode, + EnableCache, + CpuOcr); + } + + + /// + /// 根据InferenceDeviceType选择Provider + /// + /// InferenceDeviceType + /// cuda设备id + /// dml设备id + /// + /// + private static ProviderType[] GetProviderType(InferenceDeviceType inferenceDeviceType, int cudaDeviceId, + int dmlDeviceId) + { + switch (inferenceDeviceType) + { + case InferenceDeviceType.Cpu: + return [ProviderType.Cpu]; + case InferenceDeviceType.GpuDirectMl: + //只用dml不加cpu的话在很多场景下性能很差。 + return [ProviderType.Dml, ProviderType.Cpu]; + case InferenceDeviceType.Gpu: + List list = []; + SessionOptions? testSession = null; + var hasGpu = false; + if (!hasGpu && cudaDeviceId >= 0) + { + // tensorrt本身包含cuda，设备id也是cuda的id，且比纯cuda效果好很多。 + try + { + testSession = SessionOptions.MakeSessionOptionWithTensorrtProvider(cudaDeviceId); + list.Add(ProviderType.TensorRt); + hasGpu = true; + } + catch (Exception e) + { + Logger.LogDebug("[init]无法加载TensorRt。可能不支持，跳过。({Err})", e.Message); + } + finally + { + testSession?.Dispose(); + } + } + + if (!hasGpu && dmlDeviceId >= 0) + { + // dml效果不如tensorrt，但是比纯cuda稳定性强 + try + { + testSession = new SessionOptions(); + testSession.AppendExecutionProvider_DML(dmlDeviceId); + list.Add(ProviderType.Dml); + hasGpu = true; + } + catch (Exception e) + { + Logger.LogDebug("[init]无法加载DML。可能不支持，跳过。({Err})", e.Message); + } + finally + { + testSession?.Dispose(); + } + } + + if (!hasGpu && cudaDeviceId >= 0) + { + // cuda优先级比较低，因为跑起来并不太理想。 + try + { + testSession = SessionOptions.MakeSessionOptionWithCudaProvider(cudaDeviceId); + list.Add(ProviderType.Cuda); + hasGpu = true; + } + catch (Exception e) + { + Logger.LogDebug("[init]无法加载Cuda。可能不支持，跳过。({Err})", e.Message); + } + finally + { + testSession?.Dispose(); + } + } + + if (!hasGpu) + { + Logger.LogWarning("[init]GPU自动选择失败，回退到CPU处理"); + } + + //无论如何都要加入cpu，一些计算在纯gpu上不被支持或性能很烂 + list.Add(ProviderType.Cpu); + return list.ToArray(); + default: + throw new InvalidEnumArgumentException("无效的推理设备"); + } + } + + /// + /// 自动嗅探并修改path以加载cuda + /// + private static void AppendCudaPath() + { + var cudaVersion = + Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\NVIDIA Corporation\GPU Computing Toolkit\CUDA", + "FirstVersionInstalled", null)?.ToString() ?? "v12.8"; + string[] filePrefix = ["cudnn", "nvrtc", "cudart", "nvinfer", "cublas", "onnx"]; + string[] environmentVariableNames = ["PATH", "CUDA_PATH", "CUDNN_PATH", "LD_LIBRARY_PATH"]; + + // 例如: CUDNN\v9.8\lib\12.8\x64 + var validPaths = environmentVariableNames.SelectMany(s => Environment + // 获取所有可能包含CUDA/cuDNN路径的环境变量 + .GetEnvironmentVariable(s, EnvironmentVariableTarget.Process)? + .Split(Path.PathSeparator) ?? []).Distinct() + // 环境变量下层文件夹 + .SelectMany(s => + // lib路径 + [s, Path.Combine(s, cudaVersion), Path.Combine(s, "bin"), Path.Combine(s, "lib")]) + .SelectMany( + // cuda的版本 + s => cudaVersion.StartsWith("v", StringComparison.InvariantCultureIgnoreCase) + ? [s, Path.Combine(s, cudaVersion), Path.Combine(s, cudaVersion[1..])] + : [s, Path.Combine(s, cudaVersion)]) + .SelectMany(s => + { + // 体系架构 + var architecture = Enum.GetName(RuntimeInformation.ProcessArchitecture); + if (architecture is null) + { + return [s]; + } + + return + [ + s, Path.Combine(s, architecture), Path.Combine(s, architecture.ToLowerInvariant()), + Path.Combine(s, architecture.ToUpperInvariant()) + ]; + }) + .Where(basePath => !string.IsNullOrWhiteSpace(basePath)) + //构建完了需要搜索的路径，去重。 + .Distinct() + + //确定路径是否真的存在 + .Where(Directory.Exists) + .SelectMany(s => + //确定需要的文件是否存在 + filePrefix.SelectMany(se => + Directory.GetFiles(s, $"{se}*.dll").Select(Path.GetDirectoryName).WhereNotNull())) + //去重 + .Distinct(); + + AppendPath(validPaths.ToArray()); + } + + /// + /// 将附加的path应用进来 + /// + /// 附加的path字符串 + private static void AppendPath(string[] extraPath) + { + if (extraPath.Length <= 0) + { + return; + } + + var pathVariables = Environment.GetEnvironmentVariable("PATH", EnvironmentVariableTarget.Process) + ?.Split(Path.PathSeparator).ToList() ?? new List(); + pathVariables.AddRange(extraPath); + if (pathVariables.Count <= 0) + { + Logger.LogWarning("[GpuAuto]SetCudaPath:No valid paths found."); + return; + } + + var updatedPath = string.Join(Path.PathSeparator, pathVariables.Distinct()); + Logger.LogDebug("[GpuAuto]修改进程PATH为:{UpdatedPath}", updatedPath); + Environment.SetEnvironmentVariable("PATH", updatedPath, EnvironmentVariableTarget.Process); + } + + /// + /// 根据模型创建一个YoloPredictor + /// + /// 模型 + /// BgiYoloPredictor + public BgiYoloPredictor CreateYoloPredictor(BgiOnnxModel model) + { + Logger.LogDebug("[Yolo]创建yolo预测器，模型: {ModelName}", model.Name); + if (!EnableCache) + { + return new BgiYoloPredictor(model, model.ModalPath, CreateSessionOptions(model, false)); + } + + var cached = GetCached(model); + return cached == null + ? new BgiYoloPredictor(model, model.ModalPath, CreateSessionOptions(model, true)) + : new BgiYoloPredictor(model, cached, CreateSessionOptions(model, false)); + } + + /// + /// 根据模型创建一个onnx运行时的InferenceSession + /// + /// 模型 + /// 是否是用于ocr的模型，默认false + /// InferenceSession + public InferenceSession CreateInferenceSession(BgiOnnxModel model, bool ocr = false) + { + Logger.LogDebug("[ONNX]创建推理会话，模型: {ModelName}", model.Name); + ProviderType[]? providerTypes = null; + if (CpuOcr && ocr) + { + providerTypes = [ProviderType.Cpu]; + } + + if (!EnableCache) + { + return new InferenceSession(model.ModalPath, CreateSessionOptions(model, false, providerTypes)); + } + + var cached = GetCached(model); + return cached == null + ? new InferenceSession(model.ModalPath, CreateSessionOptions(model, true, providerTypes)) + : new InferenceSession(cached, CreateSessionOptions(model, false, providerTypes)); + } + + /// + /// 获取带有缓存的模型(目前只支持TensorRT) + /// + /// 模型 + /// 带有缓存的模型绝对路径，null表示尚未创建缓存 + private string? GetCached(BgiOnnxModel model) + { + // 目前只支持TensorRT + if (!ProviderTypes.Contains(ProviderType.TensorRt)) return null; + var result = _cachedModelPaths.GetOrAdd(model, _GetCached); + if (result is null) + { + return result; + } + + // 判断文件是否存在 + if (File.Exists(result)) + { + return result; + } + + Logger.LogWarning("[ONNX]模型 {Model} 的缓存文件可能已被删除，使用原始模型文件。", model.Name); + return null; + } + + private string? _GetCached(BgiOnnxModel model) + { + if (model.ModelRelativePath.StartsWith(BgiOnnxModel.ModelCacheRelativePath) && + model.ModelRelativePath.EndsWith("_ctx.onnx")) + { + // 这已经是带有缓存的文件路径了 + return model.ModalPath; + } + + var ctxA = Path.Combine(model.CachePath, "trt", "_ctx.onnx"); + if (File.Exists(ctxA)) + { + Logger.LogDebug("[ONNX]模型 {Model} 命中TRT匿名缓存文件: {Path}", model.Name, ctxA); + return ctxA; + } + + var ctxB = Path.Combine(model.CachePath, "trt", + Path.GetFileNameWithoutExtension(model.ModalPath) + "_ctx.onnx"); + if (File.Exists(ctxB)) + { + Logger.LogDebug("[ONNX]模型 {Model} 命中TRT命名缓存文件: {Path}", model.Name, ctxB); + return ctxB; + } + + Logger.LogDebug("[ONNX]没有找到模型 {Model} 的模型缓存文件。", model.Name); + return null; + } + + + /// + /// 通过模型路径生成SessionOptions
+ /// 如果加载的模型文件已经是带有缓存的模型，请将cacheFolder设为null避免重复生成。 + /// + /// 模型路径 + /// 是否生成缓存。有几种情况下不生成缓存:1为用户主动关闭，即enableCache为false。2为即将加载的模型文件已经是带有缓存的模型文件。 + /// 强制使用的Provider,为空或null则不强制 + /// + /// + private SessionOptions CreateSessionOptions(BgiOnnxModel path, bool genCache, ProviderType[]? forcedProvider = null) + { + var sessionOptions = new SessionOptions(); + foreach (var type in + forcedProvider is null || forcedProvider.Length == 0 ? ProviderTypes : forcedProvider) + { + try + { + switch (type) + { + case ProviderType.Dml: + // DirectML 执行提供程序不支持在 onnxruntime 中使用内存模式优化或并行执行。在创建 InferenceSession 期间提供会话选项时，必须禁用这些选项，否则将返回错误。 + sessionOptions.AppendExecutionProvider_DML(DmlDeviceId); + sessionOptions.EnableMemoryPattern = false; + sessionOptions.ExecutionMode = ExecutionMode.ORT_SEQUENTIAL; + break; + case ProviderType.Cpu: + sessionOptions.AppendExecutionProvider_CPU(); + break; + case ProviderType.TensorRt: + using (var options = new OrtTensorRTProviderOptions()) + { + options.UpdateOptions(GetTrtProviderConfig(genCache ? path.CachePath : null)); + sessionOptions.AppendExecutionProvider_Tensorrt(options); + } + + break; + case ProviderType.Cuda: + using (var options = new OrtCUDAProviderOptions()) + { + options.UpdateOptions(GetCudaProviderConfig()); + sessionOptions.AppendExecutionProvider_CUDA(); + } + + break; + default: + throw new InvalidEnumArgumentException("无效的推理设备"); + } + } + catch (Exception e) + { + Logger.LogError("无法加载指定的 ONNX provider {Provider}，跳过。请检查推理设备配置是否正确。({Err})", Enum.GetName(type), + e.Message); + } + } + + if (!OptimizedModel) return sessionOptions; + if (!genCache) return sessionOptions; + var optPath = Path.Combine(path.CachePath, "optimized"); + if (!Directory.Exists(optPath)) + { + Directory.CreateDirectory(optPath); + } + + sessionOptions.OptimizedModelFilePath = optPath; + return sessionOptions; + } + + + /// + /// 获取TensorRT的配置 + /// + /// 缓存生成的目录 + /// trt配置 + private Dictionary GetTrtProviderConfig(string? cacheFolder) + { + if (cacheFolder is null) + { + // 不使用缓存目录 + var r = new Dictionary + { + ["device_id"] = CudaDeviceId.ToString(), + }; + return r; + } + + var result = new Dictionary + { + ["trt_engine_cache_enable"] = "1", + ["trt_dump_ep_context_model"] = "1", + ["trt_ep_context_file_path"] = Global.Absolute(Path.Combine(cacheFolder, "trt")), + // ["trt_ep_context_embed_mode"] = "1", // 因为yoloSharp是把模型转为嵌入式运行，不这样会爆炸 + // ["trt_engine_cache_path"] = ".\\" // 没必要了 + ["trt_timing_cache_enable"] = "1", + ["trt_timing_cache_path"] = + Global.Absolute(Path.Combine(BgiOnnxModel.ModelCacheRelativePath, "trt_timing")), + // ["trt_force_timing_cache"] = "1", + ["device_id"] = CudaDeviceId.ToString(), + }; + if (TrtUseEmbedMode) + { + result["trt_ep_context_embed_mode"] = "1"; + } + else + { + result["trt_ep_context_embed_mode"] = "0"; + result["trt_engine_cache_path"] = ".\\"; + } + + if (!Directory.Exists(result["trt_ep_context_file_path"])) + { + Directory.CreateDirectory(result["trt_ep_context_file_path"]); + } + + return result; + } + + /// + /// 获取cuda provider的配置 + /// + /// cuda配置 + private Dictionary GetCudaProviderConfig() + { + var result = new Dictionary + { + ["device_id"] = CudaDeviceId.ToString(), + }; + return result; + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/BgiOnnxModel.cs b/BetterGenshinImpact/Core/Recognition/ONNX/BgiOnnxModel.cs new file mode 100644 index 00000000..e5aae73d --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/ONNX/BgiOnnxModel.cs @@ -0,0 +1,124 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using BetterGenshinImpact.Core.Config; + +namespace BetterGenshinImpact.Core.Recognition.ONNX; + +public class BgiOnnxModel +{ + /// + /// 模型使用的缓存文件的相对目录 + /// + public static readonly string ModelCacheRelativePath = Path.Combine("Cache", Global.Version, "Model"); + + private static readonly List RegisteredModels = []; + public string Name { get; private init; } + public string ModelRelativePath { get; private init; } + public string ModalPath => Global.Absolute(ModelRelativePath); + public string CacheRelativePath { get; private init; } + public string CachePath => Global.Absolute(CacheRelativePath); + + #region 模型注册 + + // 模型注册在这里，这样可以方便预先对模型预热和缓存管理等操作，避免冲突。 + // 硬编码虽然不那么优雅，但是也没想到什么好的解决办法 + /// + /// yap文字识别 + /// + public static readonly BgiOnnxModel YapModelTraining = + Register("YapModelTraining", @"Assets\Model\Yap\model_training.onnx"); + + /// + /// 钓鱼模型 + /// + public static readonly BgiOnnxModel BgiFish = Register("BgiFish", @"Assets\Model\Fish\bgi_fish.onnx"); + + /// + /// 秘境中古树 + /// + public static readonly BgiOnnxModel BgiTree = Register("BgiTree", @"Assets\Model\Domain\bgi_tree.onnx"); + + /// + /// 用于捡东西等的大世界模型 + /// + public static readonly BgiOnnxModel BgiWorld = Register("BgiTree", @"Assets\Model\World\bgi_world.onnx"); + + /// + /// 角色识别 + /// + public static readonly BgiOnnxModel BgiAvatarSide = + Register("BgiAvatarSide", @"Assets\Model\Common\avatar_side_classify_sim.onnx"); + + /// + /// paddleOCR V4 简体中文 检测模型 + /// + public static readonly BgiOnnxModel PaddleOcrChDet = + Register("ch_PP-OCRv4_det", @"Assets\Model\PaddleOCR\ch_PP-OCRv4_det\slim_model.onnx"); + + /// + /// paddleOCR V4 简体中文 识别模型 + /// + public static readonly BgiOnnxModel PaddleOcrChRec = + Register("ch_PP-OCRv4_rec", @"Assets\Model\PaddleOCR\ch_PP-OCRv4_rec\slim_model.onnx"); + + /// + /// paddleOCR V3 繁体中文 识别模型 + /// + public static readonly BgiOnnxModel PaddleOcrChtRec = + Register("chinese_cht_PP-OCRv3_rec", @"Assets\Model\PaddleOCR\chinese_cht_PP-OCRv3_rec_infer\slim_model.onnx"); + + /// + /// paddleOCR V3 英文 检测模型 + /// + public static readonly BgiOnnxModel PaddleOcrEnDet = + Register("en_PP-OCRv3_det", @"Assets\Model\PaddleOCR\en_PP-OCRv3_det_infer\slim_model.onnx"); + + /// + /// paddleOCR V3 拉丁文 识别模型 + /// + public static readonly BgiOnnxModel PaddleOcrLatinRec = + Register("latin_PP-OCRv3_rec", @"Assets\Model\PaddleOCR\latin_PP-OCRv3_rec_infer\slim_model.onnx"); + + #endregion + + private BgiOnnxModel(string name, string modelRelativePath, string cacheRelativePath) + { + Name = name; + ModelRelativePath = modelRelativePath; + CacheRelativePath = cacheRelativePath; + } + + public static bool IsModelExist(BgiOnnxModel model) + { + return File.Exists(model.ModalPath); + } + + + /// + /// 获取全部已注册的模型文件 + /// + /// + public static ImmutableList GetAll() + { + return RegisteredModels.ToImmutableList(); + } + + private static BgiOnnxModel Register(string name, string modelRelativePath) + { + return Register(name, modelRelativePath, Path.Combine(ModelCacheRelativePath, name)); + } + + private static BgiOnnxModel Register(string name, string modelRelativePath, string cacheRelativePath) + { + var model = new BgiOnnxModel(name, modelRelativePath, cacheRelativePath); + var cachePath = model.CachePath; + if (!Directory.Exists(cachePath)) + { + Directory.CreateDirectory(cachePath); + } + + RegisteredModels.Add(model); + return model; + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/BgiSessionOption.cs b/BetterGenshinImpact/Core/Recognition/ONNX/BgiSessionOption.cs deleted file mode 100644 index 6bd02057..00000000 --- a/BetterGenshinImpact/Core/Recognition/ONNX/BgiSessionOption.cs +++ /dev/null @@ -1,35 +0,0 @@ -using BetterGenshinImpact.GameTask; -using BetterGenshinImpact.Model; -using Microsoft.ML.OnnxRuntime; -using System.ComponentModel; - -namespace BetterGenshinImpact.Core.Recognition.ONNX; - -public class BgiSessionOption : Singleton -{ - public static string[] InferenceDeviceTypes { get; } = ["CPU", "GPU_DirectML"]; - - public SessionOptions Options { get; set; } = TaskContext.Instance().Config.InferenceDevice switch - { - "CPU" => new SessionOptions(), - "GPU_DirectML" => MakeSessionOptionWithDirectMlProvider(), - _ => throw new InvalidEnumArgumentException("无效的推理设备") - }; - - public static SessionOptions MakeSessionOptionWithDirectMlProvider() - { - var sessionOptions = new SessionOptions(); - sessionOptions.AppendExecutionProvider_DML(0); - return sessionOptions; - } - - // /// - // /// 重新加载每个推理器（测试没用，只能重启） - // /// - // public void RefreshInference() - // { - // // 自动秘境每次都会NEW不用管 - // // Yap、自动钓鱼 - // GameTaskManager.RefreshTriggerConfigs(); - // } -} diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloPredictor.cs b/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloPredictor.cs new file mode 100644 index 00000000..296444ef --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloPredictor.cs @@ -0,0 +1,85 @@ +using System; +using BetterGenshinImpact.GameTask.Model.Area; +using OpenCvSharp; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing.Imaging; +using System.IO; +using System.Linq; +using System.Text.Json; +using BetterGenshinImpact.View.Drawable; +using Compunet.YoloSharp; +using Microsoft.ML.OnnxRuntime; + +namespace BetterGenshinImpact.Core.Recognition.ONNX; + +public class BgiYoloPredictor : IDisposable +{ + private readonly BgiOnnxModel _model; + + + private readonly Lazy _lazyPredictor; + + /// + /// 使用 BgiOnnxFactory 创建这个类的实例 + /// + /// 模型 + /// 实际要加载的模型文件的绝对路径，在使用模型缓存的场景下可能有差别 + /// sessionOptions + protected internal BgiYoloPredictor(BgiOnnxModel onnxModel, string modelPath, SessionOptions sessionOptions) + { + _model = onnxModel; + _lazyPredictor = new Lazy(() => new YoloPredictor(modelPath, + new YoloPredictorOptions + { + SessionOptions = sessionOptions + })); + } + + public YoloPredictor Predictor => _lazyPredictor.Value; + + /// + /// 检测 + /// + /// 图像 + /// 类别-矩形框 + public Dictionary> Detect(ImageRegion region) + { + using var memoryStream = new MemoryStream(); + region.SrcBitmap.Save(memoryStream, ImageFormat.Bmp); + memoryStream.Seek(0, SeekOrigin.Begin); + var result = Predictor.Detect(memoryStream); + + + var dict = new Dictionary>(); + foreach (var box in result) + { + if (!dict.TryGetValue(box.Name.Name, out var value)) + { + dict[box.Name.Name] = [new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height)]; + } + else + { + value.Add(new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height)); + } + } + + Debug.WriteLine("YOLO识别结果:" + JsonSerializer.Serialize(dict)); + + var list = result + .Select(box => new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height)) + .Select(rect => region.ToRectDrawable(rect, _model.Name)).ToList(); + + VisionContext.Instance().DrawContent.PutOrRemoveRectList(_model.Name, list); + + return dict; + } + + public void Dispose() + { + if (_lazyPredictor.IsValueCreated) + { + Predictor.Dispose(); + } + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloV8Predictor.cs b/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloV8Predictor.cs deleted file mode 100644 index 2ce81e3a..00000000 --- a/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloV8Predictor.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System; -using BetterGenshinImpact.Core.Config; -using BetterGenshinImpact.GameTask.Model.Area; -using Compunet.YoloV8; -using OpenCvSharp; -using System.Collections.Generic; -using System.Diagnostics; -using System.Drawing.Imaging; -using System.IO; -using System.Text.Json; -using BetterGenshinImpact.View.Drawable; - -namespace BetterGenshinImpact.Core.Recognition.ONNX; - -public class BgiYoloV8Predictor(string modelRelativePath) : IDisposable -{ - private readonly YoloV8Predictor _predictor = YoloV8Builder.CreateDefaultBuilder() - .UseOnnxModel(Global.Absolute(modelRelativePath)) - .WithSessionOptions(BgiSessionOption.Instance.Options) - .Build(); - - public YoloV8Predictor Predictor => _predictor; - - /// - /// 检测 - /// - /// 图像 - /// 类别-矩形框 - public Dictionary> Detect(ImageRegion region) - { - using var memoryStream = new MemoryStream(); - region.SrcBitmap.Save(memoryStream, ImageFormat.Bmp); - memoryStream.Seek(0, SeekOrigin.Begin); - var result = _predictor.Detect(memoryStream); - - var dict = new Dictionary>(); - foreach (var box in result.Boxes) - { - if (!dict.ContainsKey(box.Class.Name)) - { - dict[box.Class.Name] = [new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height)]; - } - else - { - dict[box.Class.Name].Add(new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height)); - } - } - Debug.WriteLine("YOLOv8识别结果:" + JsonSerializer.Serialize(dict)); - - var list = new List(); - foreach (var box in result.Boxes) - { - var rect = new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height); - list.Add(region.ToRectDrawable(rect, modelRelativePath)); - } - - VisionContext.Instance().DrawContent.PutOrRemoveRectList(modelRelativePath, list); - - return dict; - } - - public void Dispose() - { - _predictor.Dispose(); - } -} diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloV8PredictorFactory.cs b/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloV8PredictorFactory.cs deleted file mode 100644 index dbad7e95..00000000 --- a/BetterGenshinImpact/Core/Recognition/ONNX/BgiYoloV8PredictorFactory.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System.Collections.Generic; - -namespace BetterGenshinImpact.Core.Recognition.ONNX; - -public class BgiYoloV8PredictorFactory -{ - static Dictionary _predictors = new(); - - public static BgiYoloV8Predictor GetPredictor(string modelRelativePath) - { - if (!_predictors.ContainsKey(modelRelativePath)) - { - _predictors[modelRelativePath] = new BgiYoloV8Predictor(modelRelativePath); - } - - return _predictors[modelRelativePath]; - } -} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/InferenceDeviceType.cs b/BetterGenshinImpact/Core/Recognition/ONNX/InferenceDeviceType.cs new file mode 100644 index 00000000..b46fdd8b --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/ONNX/InferenceDeviceType.cs @@ -0,0 +1,8 @@ +namespace BetterGenshinImpact.Core.Recognition.ONNX; + +public enum InferenceDeviceType +{ + Cpu, + GpuDirectMl, + Gpu +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/ProviderType.cs b/BetterGenshinImpact/Core/Recognition/ONNX/ProviderType.cs new file mode 100644 index 00000000..df7e0068 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/ONNX/ProviderType.cs @@ -0,0 +1,11 @@ + + +namespace BetterGenshinImpact.Core.Recognition.ONNX; + +public enum ProviderType +{ + TensorRt, + Cuda, + Dml, + Cpu +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/SVTR/PickTextInference.cs b/BetterGenshinImpact/Core/Recognition/ONNX/SVTR/PickTextInference.cs index ceb0c6c7..38fcb3df 100644 --- a/BetterGenshinImpact/Core/Recognition/ONNX/SVTR/PickTextInference.cs +++ b/BetterGenshinImpact/Core/Recognition/ONNX/SVTR/PickTextInference.cs @@ -9,6 +9,7 @@ using System.Diagnostics; using System.IO; using System.Text; using System.Text.Json; +using BetterGenshinImpact.Core.Recognition.OCR; namespace BetterGenshinImpact.Core.Recognition.ONNX.SVTR; @@ -23,23 +24,21 @@ public class PickTextInference : ITextInference public PickTextInference() { - var modelPath = Global.Absolute(@"Assets\Model\Yap\model_training.onnx"); - if (!File.Exists(modelPath)) throw new FileNotFoundException("Yap模型文件不存在", modelPath); - - _session = new InferenceSession(modelPath, BgiSessionOption.Instance.Options); + _session = BgiOnnxFactory.Instance.CreateInferenceSession(BgiOnnxModel.YapModelTraining,true); var wordJsonPath = Global.Absolute(@"Assets\Model\Yap\index_2_word.json"); if (!File.Exists(wordJsonPath)) throw new FileNotFoundException("Yap字典文件不存在", wordJsonPath); var json = File.ReadAllText(wordJsonPath); - _wordDictionary = JsonSerializer.Deserialize>(json) ?? throw new Exception("index_2_word.json deserialize failed"); + _wordDictionary = JsonSerializer.Deserialize>(json) ?? + throw new Exception("index_2_word.json deserialize failed"); } public string Inference(Mat mat) { long startTime = Stopwatch.GetTimestamp(); // 将输入数据调整为 (1, 1, 32, 384) 形状的张量 - var reshapedInputData = ToTensorUnsafe(mat, out var owner); + var reshapedInputData = OcrUtils.ToTensorYapDnn(mat, out var owner); IDisposableReadOnlyCollection results; @@ -86,6 +85,8 @@ public class PickTextInference : ITextInference } } + + [Obsolete("使用CV DNN替代")] public static Tensor ToTensorUnsafe(Mat src, out IMemoryOwner tensorMemoryOwnser) { var channels = src.Channels(); @@ -114,4 +115,4 @@ public class PickTextInference : ITextInference return new DenseTensor(memory, [1, 1, 32, 384]); } -} +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/ONNX/SVTR/TextInferenceFactory.cs b/BetterGenshinImpact/Core/Recognition/ONNX/SVTR/TextInferenceFactory.cs index 968732a9..a83620a2 100644 --- a/BetterGenshinImpact/Core/Recognition/ONNX/SVTR/TextInferenceFactory.cs +++ b/BetterGenshinImpact/Core/Recognition/ONNX/SVTR/TextInferenceFactory.cs @@ -1,4 +1,6 @@ using System; +using BetterGenshinImpact.Core.Recognition.OpenCv; +using OpenCvSharp; namespace BetterGenshinImpact.Core.Recognition.ONNX.SVTR; @@ -14,4 +16,32 @@ public class TextInferenceFactory _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), }; } + public static Mat PreProcessForInference(Mat mat) + { + if (mat.Channels() == 3) + { + mat = mat.CvtColor(ColorConversionCodes.BGR2GRAY); + } + else if (mat.Channels() == 4) + { + mat = mat.CvtColor(ColorConversionCodes.BGRA2GRAY); + } + else if (mat.Channels() != 1) + { + throw new ArgumentException("mat must be 1, 3 or 4 channels"); + } + + // Yap 已经改用灰度图了 https://github.com/Alex-Beng/Yap/commit/c2ad1e7b1442aaf2d80782a032e00876cd1c6c84 + // 二值化 + // Cv2.Threshold(mat, mat, 0, 255, ThresholdTypes.Otsu | ThresholdTypes.Binary); + //Cv2.AdaptiveThreshold(mat, mat, 255, AdaptiveThresholdTypes.GaussianC, ThresholdTypes.Binary, 31, 3); // 效果不错 但是和模型不搭 + //mat = OpenCvCommonHelper.Threshold(mat, Scalar.FromRgb(235, 235, 235), Scalar.FromRgb(255, 255, 255)); // 识别物品不太行 + // 不知道为什么要强制拉伸到 221x32 + mat = ResizeHelper.ResizeTo(mat, 221, 32); + // 填充到 384x32 + var padded = new Mat(new Size(384, 32), MatType.CV_8UC1, Scalar.Black); + padded[new Rect(0, 0, mat.Width, mat.Height)] = mat; + //Cv2.ImWrite(Global.Absolute("padded.png"), padded); + return padded; + } } diff --git a/BetterGenshinImpact/GameTask/AutoDomain/AutoDomainTask.cs b/BetterGenshinImpact/GameTask/AutoDomain/AutoDomainTask.cs index 6130505f..02a1859f 100644 --- a/BetterGenshinImpact/GameTask/AutoDomain/AutoDomainTask.cs +++ b/BetterGenshinImpact/GameTask/AutoDomain/AutoDomainTask.cs @@ -3,7 +3,6 @@ using BetterGenshinImpact.Core.Recognition.OCR; using BetterGenshinImpact.Core.Recognition.ONNX; using BetterGenshinImpact.Core.Simulator; using BetterGenshinImpact.Core.Simulator.Extensions; -using BetterGenshinImpact.GameTask.AutoFight; using BetterGenshinImpact.GameTask.AutoFight.Assets; using BetterGenshinImpact.GameTask.AutoFight.Model; using BetterGenshinImpact.GameTask.AutoFight.Script; @@ -14,7 +13,6 @@ using BetterGenshinImpact.GameTask.Model.Area; using BetterGenshinImpact.Helpers; using BetterGenshinImpact.Service.Notification; using BetterGenshinImpact.View.Drawable; -using Compunet.YoloV8; using Microsoft.Extensions.Logging; using OpenCvSharp; using System; @@ -31,14 +29,12 @@ using BetterGenshinImpact.GameTask.Common.BgiVision; using BetterGenshinImpact.GameTask.Common.Element.Assets; using BetterGenshinImpact.GameTask.Common.Job; using BetterGenshinImpact.Service.Notification.Model.Enum; -using Vanara.PInvoke; using static BetterGenshinImpact.GameTask.Common.TaskControl; -using static Vanara.PInvoke.Kernel32; -using static Vanara.PInvoke.User32; using Microsoft.Extensions.Localization; using System.Globalization; using System.Text.RegularExpressions; using BetterGenshinImpact.GameTask.AutoArtifactSalvage; +using Compunet.YoloSharp; namespace BetterGenshinImpact.GameTask.AutoDomain; @@ -48,7 +44,7 @@ public class AutoDomainTask : ISoloTask private readonly AutoDomainParam _taskParam; - private readonly YoloV8Predictor _predictor; + private readonly BgiYoloPredictor _predictor; private readonly AutoDomainConfig _config; @@ -66,10 +62,7 @@ public class AutoDomainTask : ISoloTask { AutoFightAssets.DestroyInstance(); _taskParam = taskParam; - _predictor = YoloV8Builder.CreateDefaultBuilder() - .UseOnnxModel(Global.Absolute(@"Assets\Model\Domain\bgi_tree.onnx")) - .WithSessionOptions(BgiSessionOption.Instance.Options) - .Build(); + _predictor = BgiOnnxFactory.Instance.CreateYoloPredictor(BgiOnnxModel.BgiTree); _config = TaskContext.Instance().Config.AutoDomainConfig; @@ -574,7 +567,7 @@ public class AutoDomainTask : ISoloTask using var ra = CaptureToRectArea(); var endTipsRect = ra.DeriveCrop(AutoFightAssets.Instance.EndTipsUpperRect); - var text = OcrFactory.Paddle.Ocr(endTipsRect.SrcGreyMat); + var text = OcrFactory.Paddle.Ocr(endTipsRect.SrcMat); if (Regex.IsMatch(text, this.challengeCompletedLocalizedString)) { Logger.LogInformation("检测到秘境结束提示(挑战达成)，结束秘境"); @@ -582,7 +575,7 @@ public class AutoDomainTask : ISoloTask } endTipsRect = ra.DeriveCrop(AutoFightAssets.Instance.EndTipsRect); - text = OcrFactory.Paddle.Ocr(endTipsRect.SrcGreyMat); + text = OcrFactory.Paddle.Ocr(endTipsRect.SrcMat); if (Regex.IsMatch(text, this.autoLeavingLocalizedString)) { Logger.LogInformation("检测到秘境结束提示(xxx秒后自动退出)，结束秘境"); @@ -828,9 +821,9 @@ public class AutoDomainTask : ISoloTask using var memoryStream = new MemoryStream(); region.SrcBitmap.Save(memoryStream, ImageFormat.Bmp); memoryStream.Seek(0, SeekOrigin.Begin); - var result = _predictor.Detect(memoryStream); + var result = _predictor.Predictor.Detect(memoryStream); var list = new List(); - foreach (var box in result.Boxes) + foreach (var box in result) { var rect = new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height); list.Add(region.ToRectDrawable(rect, "tree")); @@ -840,7 +833,7 @@ public class AutoDomainTask : ISoloTask if (list.Count > 0) { - var box = result.Boxes[0]; + var box = result[0]; return new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height); } @@ -1043,7 +1036,7 @@ public class AutoDomainTask : ISoloTask // 图像右侧就是脆弱树脂数量 var countArea = ra.DeriveCrop(fragileResinCountRa.X + fragileResinCountRa.Width, fragileResinCountRa.Y, (int)(fragileResinCountRa.Width * 3), fragileResinCountRa.Height); - var count = OcrFactory.Paddle.Ocr(countArea.SrcGreyMat); + var count = OcrFactory.Paddle.Ocr(countArea.SrcMat); fragileResinCount = StringUtils.TryParseInt(count); } diff --git a/BetterGenshinImpact/GameTask/AutoFight/AutoFightTask.cs b/BetterGenshinImpact/GameTask/AutoFight/AutoFightTask.cs index f20c7e42..7408a951 100644 --- a/BetterGenshinImpact/GameTask/AutoFight/AutoFightTask.cs +++ b/BetterGenshinImpact/GameTask/AutoFight/AutoFightTask.cs @@ -33,7 +33,7 @@ public class AutoFightTask : ISoloTask private CancellationToken _ct; - private readonly BgiYoloV8Predictor _predictor; + private readonly BgiYoloPredictor _predictor; private DateTime _lastFightFlagTime = DateTime.Now; // 战斗标志最近一次出现的时间 @@ -185,7 +185,7 @@ public class AutoFightTask : ISoloTask if (_taskParam.FightFinishDetectEnabled) { - _predictor = BgiYoloV8PredictorFactory.GetPredictor(@"Assets\Model\World\bgi_world.onnx"); + _predictor = BgiOnnxFactory.Instance.CreateYoloPredictor(BgiOnnxModel.BgiWorld); } _finishDetectConfig = new TaskFightFinishDetectConfig(_taskParam.FinishDetectConfig); diff --git a/BetterGenshinImpact/GameTask/AutoFight/Model/CombatScenes.cs b/BetterGenshinImpact/GameTask/AutoFight/Model/CombatScenes.cs index d4a2607b..485d69c6 100644 --- a/BetterGenshinImpact/GameTask/AutoFight/Model/CombatScenes.cs +++ b/BetterGenshinImpact/GameTask/AutoFight/Model/CombatScenes.cs @@ -1,16 +1,13 @@ -using BetterGenshinImpact.Core.Config; -using BetterGenshinImpact.Core.Recognition.OCR; +using BetterGenshinImpact.Core.Recognition.OCR; using BetterGenshinImpact.Core.Recognition.ONNX; using BetterGenshinImpact.Core.Recognition.OpenCv; using BetterGenshinImpact.GameTask.AutoFight.Assets; using BetterGenshinImpact.GameTask.AutoFight.Config; using BetterGenshinImpact.GameTask.Model.Area; using BetterGenshinImpact.Helpers; -using Compunet.YoloV8; using Microsoft.Extensions.Logging; using OpenCvSharp; using OpenCvSharp.Extensions; -using Sdcb.PaddleOCR; using System; using System.Collections.Generic; using System.Collections.ObjectModel; @@ -21,6 +18,8 @@ using System.IO; using System.Linq; using System.Threading; using BetterGenshinImpact.Core.Simulator; +using Compunet.YoloSharp; +using Compunet.YoloSharp.Data; using static BetterGenshinImpact.GameTask.Common.TaskControl; namespace BetterGenshinImpact.GameTask.AutoFight.Model; @@ -38,11 +37,8 @@ public class CombatScenes : IDisposable public int AvatarCount => Avatars.Length; - private readonly YoloV8Predictor _predictor = - YoloV8Builder.CreateDefaultBuilder() - .UseOnnxModel(Global.Absolute(@"Assets\Model\Common\avatar_side_classify_sim.onnx")) - .WithSessionOptions(BgiSessionOption.Instance.Options) - .Build(); + private readonly BgiYoloPredictor _predictor = + BgiOnnxFactory.Instance.CreateYoloPredictor(BgiOnnxModel.BgiAvatarSide); public int ExpectedTeamAvatarNum { get; private set; } = 4; @@ -167,32 +163,32 @@ public class CombatScenes : IDisposable src.Save(memoryStream, ImageFormat.Bmp); memoryStream.Seek(0, SeekOrigin.Begin); speedTimer.Record("角色侧面头像图像转换"); - var result = _predictor.Classify(memoryStream); + var result = _predictor.Predictor.Classify(memoryStream); speedTimer.Record("角色侧面头像分类识别"); Debug.WriteLine($"角色侧面头像识别结果：{result}"); speedTimer.DebugPrint(); - - if (result.TopClass.Name.Name.StartsWith("Qin") || result.TopClass.Name.Name.Contains("Costume")) + var topClass = result.GetTopClass(); + if (topClass.Name.Name.StartsWith("Qin") || topClass.Name.Name.Contains("Costume")) { // 降低琴和衣装角色的识别率要求 - if (result.TopClass.Confidence < 0.51) + if (topClass.Confidence < 0.51) { Cv2.ImWrite(@"log\avatar_side_classify_error.png", src.ToMat()); throw new Exception( - $"无法识别第{index}位角色，置信度{result.TopClass.Confidence:F1}，结果：{result.TopClass.Name.Name}。请重新阅读 BetterGI 文档中的《快速上手》！"); + $"无法识别第{index}位角色，置信度{topClass.Confidence:F1}，结果：{topClass.Name.Name}。请重新阅读 BetterGI 文档中的《快速上手》！"); } } else { - if (result.TopClass.Confidence < 0.7) + if (topClass.Confidence < 0.7) { Cv2.ImWrite(@"log\avatar_side_classify_error.png", src.ToMat()); throw new Exception( - $"无法识别第{index}位角色，置信度{result.TopClass.Confidence:F1}，结果：{result.TopClass.Name.Name}。请重新阅读 BetterGI 文档中的《快速上手》！"); + $"无法识别第{index}位角色，置信度{topClass.Confidence:F1}，结果：{topClass.Name.Name}。请重新阅读 BetterGI 文档中的《快速上手》！"); } } - return result.TopClass.Name.Name; + return topClass.Name.Name; } private void InitializeTeamFromConfig(string teamNames) @@ -393,7 +389,7 @@ public class CombatScenes : IDisposable } [Obsolete] - private void ParseTeamOcrResult(PaddleOcrResult result, ImageRegion rectArea) + private void ParseTeamOcrResult(OcrResult result, ImageRegion rectArea) { List names = []; List nameRects = []; diff --git a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs index 6ca5b765..6f668d37 100644 --- a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs +++ b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs @@ -13,7 +13,6 @@ using BetterGenshinImpact.Core.Simulator; using BetterGenshinImpact.View.Drawable; using BetterGenshinImpact.GameTask.AutoFishing.Assets; using Vanara.PInvoke; -using Compunet.YoloV8; using System.Drawing.Imaging; using System.IO; using System.Linq; @@ -29,6 +28,7 @@ using System.Globalization; using Microsoft.Extensions.Localization; using BetterGenshinImpact.Helpers; using BetterGenshinImpact.Core.Recognition.OCR; +using Compunet.YoloSharp; namespace BetterGenshinImpact.GameTask.AutoFishing { @@ -42,6 +42,9 @@ namespace BetterGenshinImpact.GameTask.AutoFishing private readonly AutoFishingTaskParam param; + private readonly BgiYoloPredictor _predictor = + BgiOnnxFactory.Instance.CreateYoloPredictor(BgiOnnxModel.BgiFish); + public AutoFishingTask(AutoFishingTaskParam param) { this.param = param; @@ -53,10 +56,11 @@ namespace BetterGenshinImpact.GameTask.AutoFishing IOcrService ocrService = OcrFactory.Paddle; - IStringLocalizer stringLocalizer = App.GetService>() ?? throw new NullReferenceException(nameof(stringLocalizer)); + IStringLocalizer stringLocalizer = + App.GetService>() ?? + throw new NullReferenceException(nameof(stringLocalizer)); - var predictor = YoloV8Builder.CreateDefaultBuilder().UseOnnxModel(Global.Absolute(@"Assets\Model\Fish\bgi_fish.onnx")).WithSessionOptions(BgiSessionOption.Instance.Options).Build(); - Blackboard blackboard = new Blackboard(predictor, this.Sleep, AutoFishingAssets.Instance); + Blackboard blackboard = new Blackboard(_predictor, this.Sleep, AutoFishingAssets.Instance); // @formatter:off var behaviourTree = FluentBuilder.Create() @@ -122,7 +126,8 @@ namespace BetterGenshinImpact.GameTask.AutoFishing // @formatter:on _logger.LogInformation("→ {Text}", "自动钓鱼，启动！"); _logger.LogWarning("请不要携带任何{Msg}，极有可能会误识别导致无法结束自动钓鱼！", "跟宠"); - _logger.LogInformation($"当前参数：{param.WholeProcessTimeoutSeconds}，{param.ThrowRodTimeOutTimeoutSeconds}，{param.FishingTimePolicy}, {param.SaveScreenshotOnKeyTick}, {param.GameCultureInfo}"); + _logger.LogInformation( + $"当前参数：{param.WholeProcessTimeoutSeconds}，{param.ThrowRodTimeOutTimeoutSeconds}，{param.FishingTimePolicy}, {param.SaveScreenshotOnKeyTick}, {param.GameCultureInfo}"); TaskContext.Instance().Config.AutoFishingConfig.Enabled = false; _logger.LogInformation("全自动运行时，自动切换实时任务中的半自动钓鱼功能为关闭状态"); @@ -139,7 +144,8 @@ namespace BetterGenshinImpact.GameTask.AutoFishing break; } - using var bitmap = TaskControl.CaptureGameImageNoRetry(TaskTriggerDispatcher.Instance().GameCapture); + using var bitmap = + TaskControl.CaptureGameImageNoRetry(TaskTriggerDispatcher.Instance().GameCapture); if (bitmap == null) { _logger.LogWarning("截图失败"); @@ -177,7 +183,9 @@ namespace BetterGenshinImpact.GameTask.AutoFishing else { SetTimeTask setTimeTask = new SetTimeTask(); - foreach (int hour in param.FishingTimePolicy == FishingTimePolicy.Daytime ? [7] : (param.FishingTimePolicy == FishingTimePolicy.Nighttime ? [19] : new int[] { 7, 19 })) + foreach (int hour in param.FishingTimePolicy == FishingTimePolicy.Daytime + ? [7] + : (param.FishingTimePolicy == FishingTimePolicy.Nighttime ? [19] : new int[] { 7, 19 })) { setTimeTask.Start(hour, 0, ct).Wait(ct); tickARound(); @@ -204,7 +212,8 @@ namespace BetterGenshinImpact.GameTask.AutoFishing /// /// /// - public WholeProcessTimeout(string name, int seconds, ILogger logger, bool saveScreenshotOnTerminate) : base(name, logger, saveScreenshotOnTerminate) + public WholeProcessTimeout(string name, int seconds, ILogger logger, bool saveScreenshotOnTerminate) : base( + name, logger, saveScreenshotOnTerminate) { this.seconds = seconds; } @@ -240,7 +249,8 @@ namespace BetterGenshinImpact.GameTask.AutoFishing /// /// 行为名将反映在提示语中 /// - public FindFishTimeout(string name, int seconds, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate) : base(name, logger, saveScreenshotOnTerminate) + public FindFishTimeout(string name, int seconds, Blackboard blackboard, ILogger logger, + bool saveScreenshotOnTerminate) : base(name, logger, saveScreenshotOnTerminate) { this.blackboard = blackboard; this.seconds = seconds; @@ -271,7 +281,8 @@ namespace BetterGenshinImpact.GameTask.AutoFishing private readonly IInputSimulator input; private readonly Blackboard blackboard; - public TurnAround(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate, IInputSimulator input) : base(name, logger, saveScreenshotOnTerminate) + public TurnAround(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate, + IInputSimulator input) : base(name, logger, saveScreenshotOnTerminate) { this.blackboard = blackboard; this.input = input; @@ -282,11 +293,12 @@ namespace BetterGenshinImpact.GameTask.AutoFishing using var memoryStream = new MemoryStream(); imageRegion.SrcBitmap.Save(memoryStream, ImageFormat.Bmp); memoryStream.Seek(0, SeekOrigin.Begin); - var result = blackboard.Predictor.Detect(memoryStream); - if (result.Boxes.Any()) + var result = blackboard.Predictor.Predictor.Detect(memoryStream); + if (result.Any()) { Fishpond fishpond = new Fishpond(result); - logger.LogInformation("定位到鱼塘：" + string.Join('、', fishpond.Fishes.GroupBy(f => f.FishType).Select(g => $"{g.Key.ChineseName}{g.Count()}条"))); + logger.LogInformation("定位到鱼塘：" + string.Join('、', + fishpond.Fishes.GroupBy(f => f.FishType).Select(g => $"{g.Key.ChineseName}{g.Count()}条"))); int i = 0; foreach (var fish in fishpond.Fishes) { @@ -350,12 +362,16 @@ namespace BetterGenshinImpact.GameTask.AutoFishing private DateTimeOffset? overallWaitEndTime; private readonly string fishingLocalizedString; - public EnterFishingMode(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate, IInputSimulator input, TimeProvider? timeProvider = null, CultureInfo? cultureInfo = null) : base(name, logger, saveScreenshotOnTerminate) + public EnterFishingMode(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate, + IInputSimulator input, TimeProvider? timeProvider = null, CultureInfo? cultureInfo = null) : base(name, + logger, saveScreenshotOnTerminate) { this.blackboard = blackboard; this.input = input; this.timeProvider = timeProvider ?? TimeProvider.System; - IStringLocalizer stringLocalizer = App.GetService>() ?? throw new NullReferenceException(nameof(stringLocalizer)); + IStringLocalizer stringLocalizer = + App.GetService>() ?? + throw new NullReferenceException(nameof(stringLocalizer)); this.fishingLocalizedString = stringLocalizer.WithCultureGet(cultureInfo, "钓鱼"); } @@ -367,13 +383,16 @@ namespace BetterGenshinImpact.GameTask.AutoFishing return BehaviourStatus.Running; } - if ((pressFWaitEndTime == null || pressFWaitEndTime < timeProvider.GetLocalNow()) && Bv.FindFAndPress(imageRegion, input.Keyboard, this.fishingLocalizedString)) + if ((pressFWaitEndTime == null || pressFWaitEndTime < timeProvider.GetLocalNow()) && + Bv.FindFAndPress(imageRegion, input.Keyboard, this.fishingLocalizedString)) { logger.LogInformation("按下钓鱼键"); pressFWaitEndTime = timeProvider.GetLocalNow().AddSeconds(3); return BehaviourStatus.Running; } - else if ((clickWhiteConfirmButtonWaitEndTime == null || clickWhiteConfirmButtonWaitEndTime < timeProvider.GetLocalNow()) && Bv.ClickWhiteConfirmButton(imageRegion)) + else if ((clickWhiteConfirmButtonWaitEndTime == null || + clickWhiteConfirmButtonWaitEndTime < timeProvider.GetLocalNow()) && + Bv.ClickWhiteConfirmButton(imageRegion)) { logger.LogInformation("点击开始钓鱼"); @@ -410,11 +429,14 @@ namespace BetterGenshinImpact.GameTask.AutoFishing private readonly Blackboard blackboard; private readonly string fishingLocalizedString; - public QuitFishingMode(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate, IInputSimulator input, CultureInfo? cultureInfo = null) : base(name, logger, saveScreenshotOnTerminate) + public QuitFishingMode(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate, + IInputSimulator input, CultureInfo? cultureInfo = null) : base(name, logger, saveScreenshotOnTerminate) { this.blackboard = blackboard; this.input = input; - IStringLocalizer stringLocalizer = App.GetService>() ?? throw new NullReferenceException(nameof(stringLocalizer)); + IStringLocalizer stringLocalizer = + App.GetService>() ?? + throw new NullReferenceException(nameof(stringLocalizer)); this.fishingLocalizedString = stringLocalizer.WithCultureGet(cultureInfo, "钓鱼"); } diff --git a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs index a6621459..2ffb19aa 100644 --- a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs +++ b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs @@ -16,7 +16,6 @@ using Fischless.WindowsInput; using BetterGenshinImpact.GameTask.Model.Area; using BetterGenshinImpact.Core.Config; using BetterGenshinImpact.Core.Recognition.ONNX; -using Compunet.YoloV8; using Microsoft.Extensions.Localization; using BetterGenshinImpact.Core.Recognition.OCR; @@ -40,6 +39,8 @@ namespace BetterGenshinImpact.GameTask.AutoFishing private Blackboard blackboard; + private readonly BgiYoloPredictor _predictor = BgiOnnxFactory.Instance.CreateYoloPredictor(BgiOnnxModel.BgiFish); + /// /// 辣条（误） /// @@ -47,23 +48,25 @@ namespace BetterGenshinImpact.GameTask.AutoFishing public AutoFishingTrigger() { - AutoFishingTaskParam autoFishingTaskParam = AutoFishingTaskParam.BuildFromConfig(TaskContext.Instance().Config.AutoFishingConfig); + AutoFishingTaskParam autoFishingTaskParam = + AutoFishingTaskParam.BuildFromConfig(TaskContext.Instance().Config.AutoFishingConfig); IOcrService ocrService = OcrFactory.Paddle; - IStringLocalizer stringLocalizer = App.GetService>() ?? throw new NullReferenceException(nameof(stringLocalizer)); - - var predictor = YoloV8Builder.CreateDefaultBuilder().UseOnnxModel(Global.Absolute(@"Assets\Model\Fish\bgi_fish.onnx")).WithSessionOptions(BgiSessionOption.Instance.Options).Build(); - this.blackboard = new Blackboard(predictor, this.Sleep, AutoFishingAssets.Instance); + IStringLocalizer stringLocalizer = + App.GetService>() ?? + throw new NullReferenceException(nameof(stringLocalizer)); + this.blackboard = new Blackboard(_predictor, this.Sleep, AutoFishingAssets.Instance); BehaviourTreeLaTiao = FluentBuilder.Create() .MySimpleParallel("root", policy: SimpleParallelPolicy.OnlyOneMustSucceed) - .Do("检查是否在钓鱼界面", CheckFishingUserInterface) - .UntilSuccess("拉条循环") - .Sequence("拉条") - .PushLeaf(() => new FishBite("自动提竿", blackboard, _logger, false, input, ocrService, cultureInfo: autoFishingTaskParam.GameCultureInfo, stringLocalizer: stringLocalizer)) - .PushLeaf(() => new GetFishBoxArea("等待拉条出现", blackboard, _logger, false)) - .PushLeaf(() => new Fishing("钓鱼拉条", blackboard, _logger, false, input)) - .End() - .End() + .Do("检查是否在钓鱼界面", CheckFishingUserInterface) + .UntilSuccess("拉条循环") + .Sequence("拉条") + .PushLeaf(() => new FishBite("自动提竿", blackboard, _logger, false, input, ocrService, + cultureInfo: autoFishingTaskParam.GameCultureInfo, stringLocalizer: stringLocalizer)) + .PushLeaf(() => new GetFishBoxArea("等待拉条出现", blackboard, _logger, false)) + .PushLeaf(() => new Fishing("钓鱼拉条", blackboard, _logger, false, input)) + .End() + .End() .End() .Build(); } @@ -199,7 +202,6 @@ namespace BetterGenshinImpact.GameTask.AutoFishing /// 1. 观察周围环境，判断鱼塘位置，视角对上鱼塘位置中心 /// 2. 根据第一步的观察结果，提前选择鱼饵 /// - [Obsolete] private (int, int) MoveMouseToFish(Rect rect1, Rect rect2) { @@ -360,6 +362,5 @@ namespace BetterGenshinImpact.GameTask.AutoFishing //{ // ClearDraw(); //} - } -} +} \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/AutoFishing/Behaviours.cs b/BetterGenshinImpact/GameTask/AutoFishing/Behaviours.cs index d578e128..4ef2ec17 100644 --- a/BetterGenshinImpact/GameTask/AutoFishing/Behaviours.cs +++ b/BetterGenshinImpact/GameTask/AutoFishing/Behaviours.cs @@ -5,7 +5,6 @@ using BetterGenshinImpact.GameTask.AutoFishing.Model; using BetterGenshinImpact.GameTask.Model.Area; using BetterGenshinImpact.Helpers; using BetterGenshinImpact.View.Drawable; -using Compunet.YoloV8; using Microsoft.Extensions.Logging; using OpenCvSharp; using System; @@ -22,6 +21,7 @@ using BetterGenshinImpact.Core.Recognition.OpenCv; using BetterGenshinImpact.Core.Simulator; using BetterGenshinImpact.GameTask.Model; using System.Globalization; +using Compunet.YoloSharp; using Microsoft.Extensions.Localization; namespace BetterGenshinImpact.GameTask.AutoFishing @@ -60,8 +60,8 @@ namespace BetterGenshinImpact.GameTask.AutoFishing using var memoryStream = new MemoryStream(); imageRegion.SrcBitmap.Save(memoryStream, ImageFormat.Bmp); memoryStream.Seek(0, SeekOrigin.Begin); - var result = blackboard.Predictor.Detect(memoryStream); - Debug.WriteLine($"YOLOv8识别: {result.Speed}"); + var result = blackboard.Predictor.Predictor.Detect(memoryStream); + Debug.WriteLine($"YOLO识别: {result.Speed}"); var fishpond = new Fishpond(result, ignoreObtained: true); if (fishpond.FishpondRect == default) { @@ -308,7 +308,7 @@ namespace BetterGenshinImpact.GameTask.AutoFishing using var memoryStream = new MemoryStream(); imageRegion.SrcBitmap.Save(memoryStream, ImageFormat.Bmp); memoryStream.Seek(0, SeekOrigin.Begin); - var result = blackboard.Predictor.Detect(memoryStream); + var result = blackboard.Predictor.Predictor.Detect(memoryStream); Debug.WriteLine($"YOLOv8识别: {result.Speed}"); var fishpond = new Fishpond(result, includeTarget: timeProvider.GetLocalNow() <= ignoreObtainedEndTime); blackboard.fishpond = fishpond; @@ -733,8 +733,7 @@ namespace BetterGenshinImpact.GameTask.AutoFishing } // OCR 提竿判断 - using Mat wordCaptureGreyMat = new Mat(imageRegion.SrcGreyMat, liftingWordsAreaRect); - var text = ocrService.Ocr(wordCaptureGreyMat); + var text = ocrService.Ocr(wordCaptureMat); if (!string.IsNullOrEmpty(text) && StringUtils.RemoveAllSpace(text).Contains(this.getABiteLocalizedString)) { diff --git a/BetterGenshinImpact/GameTask/AutoFishing/Blackboard.cs b/BetterGenshinImpact/GameTask/AutoFishing/Blackboard.cs index 40ae7960..f460772f 100644 --- a/BetterGenshinImpact/GameTask/AutoFishing/Blackboard.cs +++ b/BetterGenshinImpact/GameTask/AutoFishing/Blackboard.cs @@ -1,9 +1,8 @@ using System; using System.Collections.Generic; -using System.Text; +using BetterGenshinImpact.Core.Recognition.ONNX; using BetterGenshinImpact.GameTask.AutoFishing.Assets; using BetterGenshinImpact.GameTask.AutoFishing.Model; -using Compunet.YoloV8; using OpenCvSharp; namespace BetterGenshinImpact.GameTask.AutoFishing @@ -73,8 +72,8 @@ namespace BetterGenshinImpact.GameTask.AutoFishing internal bool pitchReset = true; #region 分层暂放 - private readonly YoloV8Predictor? predictor; - internal YoloV8Predictor Predictor + private readonly BgiYoloPredictor? predictor; + internal BgiYoloPredictor Predictor { get { @@ -93,7 +92,7 @@ namespace BetterGenshinImpact.GameTask.AutoFishing } - public Blackboard(YoloV8Predictor? predictor = null, Action? sleep = null, AutoFishingAssets? autoFishingAssets = null) + public Blackboard(BgiYoloPredictor? predictor = null, Action? sleep = null, AutoFishingAssets? autoFishingAssets = null) { this.predictor = predictor; this.Sleep = sleep ?? (_ => throw new NotImplementedException()); diff --git a/BetterGenshinImpact/GameTask/AutoFishing/Model/Fishpond.cs b/BetterGenshinImpact/GameTask/AutoFishing/Model/Fishpond.cs index ec5e7bdd..47898985 100644 --- a/BetterGenshinImpact/GameTask/AutoFishing/Model/Fishpond.cs +++ b/BetterGenshinImpact/GameTask/AutoFishing/Model/Fishpond.cs @@ -1,10 +1,8 @@ -using BetterGenshinImpact.Core.Recognition.OpenCv; -using Compunet.YoloV8.Data; -using OpenCvSharp; -using System; +using OpenCvSharp; using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Compunet.YoloSharp.Data; namespace BetterGenshinImpact.GameTask.AutoFishing.Model; @@ -37,10 +35,10 @@ public class Fishpond /// /// 是否包含抛竿落点 /// 是否忽略“获得”物品的图标 - public Fishpond(DetectionResult result, bool includeTarget = false, bool ignoreObtained = false) + public Fishpond(YoloResult result, bool includeTarget = false, bool ignoreObtained = false) { Print(result); - foreach (var box in result.Boxes) + foreach (var box in result) { // 可信度太低的直接放弃 if (box.Confidence < 0.4) @@ -49,7 +47,7 @@ public class Fishpond } Rect rect = new Rect(box.Bounds.X, box.Bounds.Y, box.Bounds.Width, box.Bounds.Height); - if (box.Class.Name == "rod" || box.Class.Name == "err rod") + if (box.Name.Name == "rod" || box.Name.Name == "err rod") { TargetRect = rect; continue; @@ -60,19 +58,19 @@ public class Fishpond // todo：不是很重要但有机会可以从构造函数里分离逻辑 // 忽略界面左侧提示的“获得”物品的图标，当上一竿获得鱼时，会对当前竿产生干扰 // 使用估算大小和位置的方式来判断并剔除 - if (box.Bounds.Width < result.Image.Width * 0.036 && box.Bounds.Height < result.Image.Width * 0.036) + if (box.Bounds.Width < result.ImageSize.Width * 0.036 && box.Bounds.Height < result.ImageSize.Width * 0.036) { - Rect huode = new Rect((int)(0.04375 * result.Image.Width), (int)(0.4666 * result.Image.Height), (int)(0.1 * result.Image.Width), (int)(0.1 * result.Image.Width)); + Rect huode = new Rect((int)(0.04375 * result.ImageSize.Width), (int)(0.4666 * result.ImageSize.Height), (int)(0.1 * result.ImageSize.Width), (int)(0.1 * result.ImageSize.Width)); if (huode.Contains(rect)) { continue; } } // 忽略界面中央提示的“获得”物品的图标 - if (box.Bounds.Width > result.Image.Width * 0.03 && box.Bounds.Width < result.Image.Width * 0.06 && - box.Bounds.Height > result.Image.Width * 0.03 && box.Bounds.Height < result.Image.Width * 0.06) + if (box.Bounds.Width > result.ImageSize.Width * 0.03 && box.Bounds.Width < result.ImageSize.Width * 0.06 && + box.Bounds.Height > result.ImageSize.Width * 0.03 && box.Bounds.Height < result.ImageSize.Width * 0.06) { - Rect huode = new Rect((int)(0.4 * result.Image.Width), (int)(0.445 * result.Image.Height), (int)(0.2 * result.Image.Width), (int)(0.06125 * result.Image.Width)); + Rect huode = new Rect((int)(0.4 * result.ImageSize.Width), (int)(0.445 * result.ImageSize.Height), (int)(0.2 * result.ImageSize.Width), (int)(0.06125 * result.ImageSize.Width)); if (huode.Contains(rect)) { continue; @@ -81,13 +79,13 @@ public class Fishpond } if (includeTarget) { - if (box.Class.Name == "koi") //进入抛竿的时候只看koihead + if (box.Name.Name == "koi") //进入抛竿的时候只看koihead { continue; } } - var fish = new OneFish(box.Class.Name, rect, box.Confidence); + var fish = new OneFish(box.Name.Name, rect, box.Confidence); Fishes.Add(fish); } @@ -97,10 +95,10 @@ public class Fishpond FishpondRect = CalculateFishpondRect(); } - private void Print(DetectionResult result) + private void Print(YoloResult result) { Debug.Write("鱼塘YOLO识别结果："); - foreach (var box in result.Boxes) + foreach (var box in result) { Debug.Write(box.ToString()); } diff --git a/BetterGenshinImpact/GameTask/AutoGeniusInvokation/GeniusInvokationControl.cs b/BetterGenshinImpact/GameTask/AutoGeniusInvokation/GeniusInvokationControl.cs index f64bf2d5..3a587842 100644 --- a/BetterGenshinImpact/GameTask/AutoGeniusInvokation/GeniusInvokationControl.cs +++ b/BetterGenshinImpact/GameTask/AutoGeniusInvokation/GeniusInvokationControl.cs @@ -107,13 +107,13 @@ public class GeniusInvokationControl { return; } + TaskControl.TrySuspend(); if (!SystemControl.IsGenshinImpactActiveByProcess()) { _logger.LogWarning("当前获取焦点的窗口不是原神，暂停"); throw new RetryException("当前获取焦点的窗口不是原神"); } - }, TimeSpan.FromSeconds(1), 100); if (_ct is { IsCancellationRequested: true }) @@ -162,7 +162,8 @@ public class GeniusInvokationControl }) .ToDictionary(x => x.Key, x => x.Value); // 打印排序后的顺序 - var msg = _assets.ActionPhaseDiceMats.Aggregate("", (current, kvp) => current + $"{kvp.Key.ToElementalType().ToChinese()}| "); + var msg = _assets.ActionPhaseDiceMats.Aggregate("", + (current, kvp) => current + $"{kvp.Key.ToElementalType().ToChinese()}| "); _logger.LogDebug("当前骰子排序：{Msg}", msg); } @@ -359,7 +360,8 @@ public class GeniusInvokationControl return dictionary; }*/ - public static Dictionary> FindMultiPicFromOneImage2OneByOne(Mat srcMat, Dictionary imgSubDictionary, double threshold = 0.8) + public static Dictionary> FindMultiPicFromOneImage2OneByOne(Mat srcMat, + Dictionary imgSubDictionary, double threshold = 0.8) { var dictionary = new Dictionary>(); foreach (var kvp in imgSubDictionary) @@ -368,11 +370,13 @@ public class GeniusInvokationControl while (true) { - var point = MatchTemplateHelper.MatchTemplate(srcMat, kvp.Value, TemplateMatchModes.CCoeffNormed, null, threshold); + var point = MatchTemplateHelper.MatchTemplate(srcMat, kvp.Value, TemplateMatchModes.CCoeffNormed, null, + threshold); if (point != new Point()) { // 把结果给遮掩掉，避免重复识别 - Cv2.Rectangle(srcMat, point, new Point(point.X + kvp.Value.Width, point.Y + kvp.Value.Height), Scalar.Black, -1); + Cv2.Rectangle(srcMat, point, new Point(point.X + kvp.Value.Width, point.Y + kvp.Value.Height), + Scalar.Black, -1); list.Add(point); } else @@ -442,7 +446,8 @@ public class GeniusInvokationControl // 选中重投 foreach (var point in kvp.Value) { - ClickCaptureArea(point.X + _assets.RollPhaseDiceMats[kvp.Key].Width / 2, point.Y + _assets.RollPhaseDiceMats[kvp.Key].Height / 2); + ClickCaptureArea(point.X + _assets.RollPhaseDiceMats[kvp.Key].Width / 2, + point.Y + _assets.RollPhaseDiceMats[kvp.Key].Height / 2); Sleep(100); } } @@ -471,7 +476,8 @@ public class GeniusInvokationControl // 3.重投骰子 _logger.LogInformation("等待5s投骰动画..."); - var msg = holdElementalTypes.Aggregate(" ", (current, elementalType) => current + (elementalType.ToChinese() + " ")); + var msg = holdElementalTypes.Aggregate(" ", + (current, elementalType) => current + (elementalType.ToChinese() + " ")); _logger.LogInformation("保留{Msg}骰子", msg); Sleep(5000); @@ -520,7 +526,8 @@ public class GeniusInvokationControl var srcMat = CaptureGameMat(); Cv2.CvtColor(srcMat, srcMat, ColorConversionCodes.BGRA2BGR); // 切割图片后再识别 加快速度 位置没啥用，所以切割后比较方便 - var dictionary = FindMultiPicFromOneImage2OneByOne(CutRight(srcMat, srcMat.Width / 5), _assets.ActionPhaseDiceMats, 0.7); + var dictionary = + FindMultiPicFromOneImage2OneByOne(CutRight(srcMat, srcMat.Width / 5), _assets.ActionPhaseDiceMats, 0.7); var msg = ""; var result = new Dictionary(); @@ -667,16 +674,19 @@ public class GeniusInvokationControl } } - int needSpecifyElementDiceCount = diceCost - diceStatus[ElementalType.Omni.ToLowerString()] - diceStatus[elementalType.ToLowerString()]; + int needSpecifyElementDiceCount = diceCost - diceStatus[ElementalType.Omni.ToLowerString()] - + diceStatus[elementalType.ToLowerString()]; if (needSpecifyElementDiceCount > 0) { if (duel.CurrentCardCount < needSpecifyElementDiceCount) { - _logger.LogInformation("当前手牌数{Current}小于需要烧牌数量{Expect}，无法释放技能", duel.CurrentCardCount, needSpecifyElementDiceCount); + _logger.LogInformation("当前手牌数{Current}小于需要烧牌数量{Expect}，无法释放技能", duel.CurrentCardCount, + needSpecifyElementDiceCount); return false; } - _logger.LogInformation("当前需要的元素骰子数量不足{Cost}个，还缺{Lack}个，当前手牌数{Current}，烧牌", diceCost, needSpecifyElementDiceCount, duel.CurrentCardCount); + _logger.LogInformation("当前需要的元素骰子数量不足{Cost}个，还缺{Lack}个，当前手牌数{Current}，烧牌", diceCost, + needSpecifyElementDiceCount, duel.CurrentCardCount); for (var i = 0; i < needSpecifyElementDiceCount; i++) { @@ -807,7 +817,8 @@ public class GeniusInvokationControl { for (var i = 0; i < rects.Count; i++) { - if (IsOverlap(rects[i], new Rect(p.X, p.Y, _assets.CharacterDefeatedMat.Width, _assets.CharacterDefeatedMat.Height))) + if (IsOverlap(rects[i], + new Rect(p.X, p.Y, _assets.CharacterDefeatedMat.Width, _assets.CharacterDefeatedMat.Height))) { res[i] = true; } @@ -999,20 +1010,23 @@ public class GeniusInvokationControl character.Area.Width + 40, character.Area.Height + 10)); // 识别角色异常状态 - var pCharacterStatusFreeze = MatchTemplateHelper.MatchTemplate(characterMat, _assets.CharacterStatusFreezeMat, TemplateMatchModes.CCoeffNormed); + var pCharacterStatusFreeze = MatchTemplateHelper.MatchTemplate(characterMat, _assets.CharacterStatusFreezeMat, + TemplateMatchModes.CCoeffNormed); if (pCharacterStatusFreeze != new Point()) { character.StatusList.Add(CharacterStatusEnum.Frozen); } - var pCharacterStatusDizziness = MatchTemplateHelper.MatchTemplate(characterMat, _assets.CharacterStatusDizzinessMat, TemplateMatchModes.CCoeffNormed); + var pCharacterStatusDizziness = MatchTemplateHelper.MatchTemplate(characterMat, + _assets.CharacterStatusDizzinessMat, TemplateMatchModes.CCoeffNormed); if (pCharacterStatusDizziness != new Point()) { character.StatusList.Add(CharacterStatusEnum.Frozen); } // 识别角色能量 - var energyPointList = MatchTemplateHelper.MatchTemplateMulti(characterMat.Clone(), _assets.CharacterEnergyOnMat, 0.8); + var energyPointList = + MatchTemplateHelper.MatchTemplateMulti(characterMat.Clone(), _assets.CharacterEnergyOnMat, 0.8); character.EnergyByRecognition = energyPointList.Count; character.Hp = hp; @@ -1048,7 +1062,8 @@ public class GeniusInvokationControl var highPurple = new Scalar(255, 255, 255); Mat gray = OpenCvCommonHelper.Threshold(bottomMat, lowPurple, highPurple); - var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new OpenCvSharp.Size(15, 10), new OpenCvSharp.Point(-1, -1)); + var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new OpenCvSharp.Size(15, 10), + new OpenCvSharp.Point(-1, -1)); Cv2.Dilate(gray, gray, kernel); //膨胀 Cv2.FindContours(gray, out var contours, out _, RetrievalModes.External, @@ -1108,7 +1123,7 @@ public class GeniusInvokationControl throw new System.Exception("未能获取到我方角色卡位置"); } - var srcMat = CaptureGameGreyMat(); + var imageRegion = CaptureToRectArea(); var hpArray = new int[3]; // 1 代表未出战 2 代表出战 for (var i = 0; i < duel.CharacterCardRects.Count; i++) @@ -1122,7 +1137,7 @@ public class GeniusInvokationControl var cardRect = duel.CharacterCardRects[i]; // 未出战角色的hp区域 - var hpMat = new Mat(srcMat, new Rect(cardRect.X + _config.CharacterCardExtendHpRect.X, + var hpMat = new Mat(imageRegion.SrcMat, new Rect(cardRect.X + _config.CharacterCardExtendHpRect.X, cardRect.Y + _config.CharacterCardExtendHpRect.Y, _config.CharacterCardExtendHpRect.Width, _config.CharacterCardExtendHpRect.Height)); var text = OcrFactory.Paddle.Ocr(hpMat); @@ -1135,7 +1150,7 @@ public class GeniusInvokationControl } else { - hpMat = new Mat(srcMat, new Rect(cardRect.X + _config.CharacterCardExtendHpRect.X, + hpMat = new Mat(imageRegion.SrcMat, new Rect(cardRect.X + _config.CharacterCardExtendHpRect.X, cardRect.Y + _config.CharacterCardExtendHpRect.Y - _config.ActiveCharacterCardSpace, _config.CharacterCardExtendHpRect.Width, _config.CharacterCardExtendHpRect.Height)); text = OcrFactory.Paddle.Ocr(hpMat); @@ -1151,7 +1166,7 @@ public class GeniusInvokationControl hpArray[i] = 2; duel.CurrentCharacter = duel.Characters[i + 1]; - AppendCharacterStatus(duel.CurrentCharacter, srcMat, hp); + AppendCharacterStatus(duel.CurrentCharacter, imageRegion.SrcGreyMat, hp); return duel.CurrentCharacter; } } @@ -1163,7 +1178,7 @@ public class GeniusInvokationControl var index = hpArray.ToList().FindIndex(x => x != 1); Debug.WriteLine($"通过OCR HP的方式没有识别到出战角色，但是通过排除法确认角色{index + 1}处于出战状态！"); duel.CurrentCharacter = duel.Characters[index + 1]; - AppendCharacterStatus(duel.CurrentCharacter, srcMat); + AppendCharacterStatus(duel.CurrentCharacter, imageRegion.SrcGreyMat); return duel.CurrentCharacter; } @@ -1239,4 +1254,4 @@ public class GeniusInvokationControl return -10; } } -} +} \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/AutoPathing/PathExecutor.cs b/BetterGenshinImpact/GameTask/AutoPathing/PathExecutor.cs index 4a1703af..8dc37bd9 100644 --- a/BetterGenshinImpact/GameTask/AutoPathing/PathExecutor.cs +++ b/BetterGenshinImpact/GameTask/AutoPathing/PathExecutor.cs @@ -632,7 +632,7 @@ public class PathExecutor var ra1 = CaptureToRectArea(); var textRect = new Rect(60, 20, 160, 260); - var textMat = new Mat(ra1.SrcGreyMat, textRect); + var textMat = new Mat(ra1.SrcMat, textRect); string text = OcrFactory.Paddle.Ocr(textMat); if (text.Contains("探索派遣奖励")) { diff --git a/BetterGenshinImpact/GameTask/AutoPick/AutoPickTrigger.cs b/BetterGenshinImpact/GameTask/AutoPick/AutoPickTrigger.cs index ec13dded..ea954702 100644 --- a/BetterGenshinImpact/GameTask/AutoPick/AutoPickTrigger.cs +++ b/BetterGenshinImpact/GameTask/AutoPick/AutoPickTrigger.cs @@ -109,6 +109,7 @@ public partial class AutoPickTrigger : ITaskTrigger { Thread.Sleep(1000); } + var speedTimer = new SpeedTimer(); using var foundRectArea = content.CaptureRectArea.Find(_pickRo); @@ -185,8 +186,9 @@ public partial class AutoPickTrigger : ITaskTrigger return; } - var textMat = new Mat(content.CaptureRectArea.SrcGreyMat, textRect); - var gradMat = new Mat(textMat, new Rect(0, 0, textRect.Width, Math.Min(textRect.Height, 3))); + // var textMat = new Mat(content.CaptureRectArea.SrcGreyMat, textRect); + var gradMat = new Mat(content.CaptureRectArea.SrcGreyMat, + new Rect(textRect.X, textRect.Y, textRect.Width, Math.Min(textRect.Height, 3))); var avgGrad = gradMat.Sobel(MatType.CV_32F, 1, 0).Mean().Val0; if (avgGrad < -3) { @@ -197,11 +199,12 @@ public partial class AutoPickTrigger : ITaskTrigger string text; if (config.OcrEngine == PickOcrEngineEnum.Yap.ToString()) { - var paddedMat = PreProcessForInference(textMat); - text = _pickTextInference.Inference(paddedMat); + var textMat = new Mat(content.CaptureRectArea.SrcGreyMat, textRect); + text = _pickTextInference.Inference(TextInferenceFactory.PreProcessForInference(textMat)); } else { + var textMat = new Mat(content.CaptureRectArea.SrcMat, textRect); text = OcrFactory.Paddle.Ocr(textMat); } @@ -280,22 +283,6 @@ public partial class AutoPickTrigger : ITaskTrigger } - private Mat PreProcessForInference(Mat mat) - { - // Yap 已经改用灰度图了 https://github.com/Alex-Beng/Yap/commit/c2ad1e7b1442aaf2d80782a032e00876cd1c6c84 - // 二值化 - // Cv2.Threshold(mat, mat, 0, 255, ThresholdTypes.Otsu | ThresholdTypes.Binary); - //Cv2.AdaptiveThreshold(mat, mat, 255, AdaptiveThresholdTypes.GaussianC, ThresholdTypes.Binary, 31, 3); // 效果不错 但是和模型不搭 - //mat = OpenCvCommonHelper.Threshold(mat, Scalar.FromRgb(235, 235, 235), Scalar.FromRgb(255, 255, 255)); // 识别物品不太行 - // 不知道为什么要强制拉伸到 221x32 - mat = ResizeHelper.ResizeTo(mat, 221, 32); - // 填充到 384x32 - var padded = new Mat(new Size(384, 32), MatType.CV_8UC1, Scalar.Black); - padded[new Rect(0, 0, mat.Width, mat.Height)] = mat; - //Cv2.ImWrite(Global.Absolute("padded.png"), padded); - return padded; - } - /// /// 相同文字前后3帧内只输出一次 /// diff --git a/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs b/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs index 03dd0ff4..c7ef6e79 100644 --- a/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs +++ b/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs @@ -305,7 +305,7 @@ public partial class AutoSkipTrigger : ITaskTrigger // OCR识别选项文字 foreach (var hangoutOption in hangoutOptionList) { - var text = OcrFactory.Paddle.Ocr(hangoutOption.TextRect!.SrcGreyMat); + var text = OcrFactory.Paddle.Ocr(hangoutOption.TextRect!.SrcMat); hangoutOption.OptionTextSrc = StringUtils.RemoveAllEnter(text); } diff --git a/BetterGenshinImpact/GameTask/AutoSkip/ExpeditionTask.cs b/BetterGenshinImpact/GameTask/AutoSkip/ExpeditionTask.cs index f279e64b..7acffa66 100644 --- a/BetterGenshinImpact/GameTask/AutoSkip/ExpeditionTask.cs +++ b/BetterGenshinImpact/GameTask/AutoSkip/ExpeditionTask.cs @@ -4,7 +4,6 @@ using BetterGenshinImpact.GameTask.Common; using BetterGenshinImpact.View.Drawable; using Microsoft.Extensions.Logging; using OpenCvSharp; -using Sdcb.PaddleOCR; using System; using System.Collections.Generic; using System.Drawing; @@ -61,7 +60,8 @@ public class ExpeditionTask ExpeditionCharacterList.Clear(); str = str.Replace("，", ","); str.Split(',').ToList().ForEach(x => ExpeditionCharacterList.Add(x.Trim())); - TaskContext.Instance().Config.AutoSkipConfig.AutoReExploreCharacter = string.Join(",", ExpeditionCharacterList); + TaskContext.Instance().Config.AutoSkipConfig.AutoReExploreCharacter = + string.Join(",", ExpeditionCharacterList); } } @@ -72,13 +72,16 @@ public class ExpeditionTask for (var i = 0; i < 5; i++) { - var result = CaptureAndOcr(content, new Rect(0, 0, captureRect.Width - (int)(480 * assetScale), captureRect.Height)); + var result = CaptureAndOcr(content, + new Rect(0, 0, captureRect.Width - (int)(480 * assetScale), captureRect.Height)); var rect = result.FindRectByText("探险完成"); // TODO i>1 的时候,可以通过关键词“探索派遣限制 4 / 5 ”判断是否已经派遣完成？ if (rect != default) { // 点击探险完成下方的人物头像 - content.CaptureRectArea.Derive(new Rect(rect.X, rect.Y + (int)(50 * assetScale), rect.Width, (int)(80 * assetScale))).Click(); + content.CaptureRectArea + .Derive(new Rect(rect.X, rect.Y + (int)(50 * assetScale), rect.Width, (int)(80 * assetScale))) + .Click(); TaskControl.Sleep(100); // 重新截图 找领取 result = CaptureAndOcr(content); @@ -128,7 +131,8 @@ public class ExpeditionTask var cards = GetCharacterCards(result); if (cards.Count > 0) { - var card = cards.FirstOrDefault(c => c.Idle && c.Name != null && ExpeditionCharacterList.Contains(c.Name)) ?? cards.First(c => c.Idle); + var card = cards.FirstOrDefault(c => + c.Idle && c.Name != null && ExpeditionCharacterList.Contains(c.Name)) ?? cards.First(c => c.Idle); var rect = card.Rects.First(); using var ra = content.CaptureRectArea.Derive(rect); @@ -147,7 +151,7 @@ public class ExpeditionTask /// /// /// - private List GetCharacterCards(PaddleOcrResult result) + private List GetCharacterCards(OcrResult result) { var captureRect = TaskContext.Instance().SystemInfo.CaptureAreaRect; var assetScale = TaskContext.Instance().SystemInfo.AssetScale; @@ -162,7 +166,8 @@ public class ExpeditionTask var cards = new List(); foreach (var ocrResultRect in ocrResultRects) { - if (ocrResultRect.Text.Contains("时间缩短") || ocrResultRect.Text.Contains("奖励增加") || ocrResultRect.Text.Contains("暂无加成")) + if (ocrResultRect.Text.Contains("时间缩短") || ocrResultRect.Text.Contains("奖励增加") || + ocrResultRect.Text.Contains("暂无加成")) { var card = new ExpeditionCharacterCard(); card.Rects.Add(ocrResultRect.Rect); @@ -170,18 +175,21 @@ public class ExpeditionTask foreach (var ocrResultRect2 in ocrResultRects) { if (ocrResultRect2.Rect.Y > ocrResultRect.Rect.Y - 50 * assetScale - && ocrResultRect2.Rect.Y + ocrResultRect2.Rect.Height < ocrResultRect.Rect.Y + ocrResultRect.Rect.Height) + && ocrResultRect2.Rect.Y + ocrResultRect2.Rect.Height < + ocrResultRect.Rect.Y + ocrResultRect.Rect.Height) { if (ocrResultRect2.Text.Contains("探险完成") || ocrResultRect2.Text.Contains("探险中")) { card.Idle = false; - var name = ocrResultRect2.Text.Replace("探险完成", "").Replace("探险中", "").Replace("/", "").Trim(); + var name = ocrResultRect2.Text.Replace("探险完成", "").Replace("探险中", "").Replace("/", "") + .Trim(); if (!string.IsNullOrEmpty(name)) { card.Name = name; } } - else if (!ocrResultRect2.Text.Contains("时间缩短") && !ocrResultRect2.Text.Contains("奖励增加") && !ocrResultRect2.Text.Contains("暂无加成")) + else if (!ocrResultRect2.Text.Contains("时间缩短") && !ocrResultRect2.Text.Contains("奖励增加") && + !ocrResultRect2.Text.Contains("暂无加成")) { card.Name = ocrResultRect2.Text; } @@ -206,7 +214,7 @@ public class ExpeditionTask private readonly Pen _pen = new(Color.Red, 1); - private PaddleOcrResult CaptureAndOcr(CaptureContent content) + private OcrResult CaptureAndOcr(CaptureContent content) { using var ra = TaskControl.CaptureToRectArea(); var result = OcrFactory.Paddle.OcrResult(ra.SrcGreyMat); @@ -214,11 +222,11 @@ public class ExpeditionTask return result; } - private PaddleOcrResult CaptureAndOcr(CaptureContent content, Rect rect) + private OcrResult CaptureAndOcr(CaptureContent content, Rect rect) { using var ra = TaskControl.CaptureToRectArea(); var result = OcrFactory.Paddle.OcrResult(ra.SrcGreyMat); //VisionContext.Instance().DrawContent.PutOrRemoveRectList("OcrResultRects", result.ToRectDrawableList(_pen)); return result; } -} +} \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/AutoWood/AutoWoodTask.cs b/BetterGenshinImpact/GameTask/AutoWood/AutoWoodTask.cs index 0b29ca9a..70ab29f3 100644 --- a/BetterGenshinImpact/GameTask/AutoWood/AutoWoodTask.cs +++ b/BetterGenshinImpact/GameTask/AutoWood/AutoWoodTask.cs @@ -228,7 +228,7 @@ public partial class AutoWoodTask : ISoloTask { // OCR识别文本区域 var woodCountRect = CaptureToRectArea().DeriveCrop(assert.WoodCountUpperRect); - return OcrFactory.Paddle.Ocr(woodCountRect.SrcGreyMat); + return OcrFactory.Paddle.Ocr(woodCountRect.SrcMat); } private bool HasDetectedWoodText(string recognizedText) diff --git a/BetterGenshinImpact/GameTask/Common/Job/ScanPickTask.cs b/BetterGenshinImpact/GameTask/Common/Job/ScanPickTask.cs index cd28b86a..a7f70f92 100644 --- a/BetterGenshinImpact/GameTask/Common/Job/ScanPickTask.cs +++ b/BetterGenshinImpact/GameTask/Common/Job/ScanPickTask.cs @@ -22,7 +22,7 @@ namespace BetterGenshinImpact.GameTask.Common.Job; /// public class ScanPickTask { - private readonly BgiYoloV8Predictor _predictor = BgiYoloV8PredictorFactory.GetPredictor(@"Assets\Model\World\bgi_world.onnx"); + private readonly BgiYoloPredictor _predictor = BgiOnnxFactory.Instance.CreateYoloPredictor(BgiOnnxModel.BgiWorld); private readonly double _dpi = TaskContext.Instance().DpiScale; private readonly RECT _realCaptureRect = TaskContext.Instance().SystemInfo.CaptureAreaRect; diff --git a/BetterGenshinImpact/GameTask/Model/Area/ImageRegion.cs b/BetterGenshinImpact/GameTask/Model/Area/ImageRegion.cs index b896e5ad..a57e7ff7 100644 --- a/BetterGenshinImpact/GameTask/Model/Area/ImageRegion.cs +++ b/BetterGenshinImpact/GameTask/Model/Area/ImageRegion.cs @@ -226,10 +226,10 @@ public class ImageRegion : Region throw new Exception($"[OCR]识别对象{ro.Name}的匹配文本不能全为空"); } - var roi = SrcGreyMat; + var roi = SrcMat; if (ro.RegionOfInterest != default) { - roi = new Mat(SrcGreyMat, ro.RegionOfInterest); + roi = new Mat(SrcMat, ro.RegionOfInterest); } var result = OcrFactory.Paddle.OcrResult(roi); @@ -323,10 +323,10 @@ public class ImageRegion : Region } else { - roi = SrcGreyMat; + roi = SrcMat; if (ro.RegionOfInterest != default) { - roi = new Mat(SrcGreyMat, ro.RegionOfInterest); + roi = new Mat(SrcMat, ro.RegionOfInterest); } } @@ -454,10 +454,10 @@ public class ImageRegion : Region } else if (RecognitionTypes.Ocr.Equals(ro.RecognitionType)) { - var roi = SrcGreyMat; + var roi = SrcMat; if (ro.RegionOfInterest != default) { - roi = new Mat(SrcGreyMat, ro.RegionOfInterest); + roi = new Mat(SrcMat, ro.RegionOfInterest); } var result = OcrFactory.Paddle.OcrResult(roi); diff --git a/BetterGenshinImpact/View/MainWindow.xaml b/BetterGenshinImpact/View/MainWindow.xaml index ee303e8c..8d579bfc 100644 --- a/BetterGenshinImpact/View/MainWindow.xaml +++ b/BetterGenshinImpact/View/MainWindow.xaml @@ -177,7 +177,7 @@ - + - 截图器启动后才能使用各项功能， - + 截图器启动后才能使用各项功能， 点击展开启动相关配置 - 。 - + 。 + ItemsSource="{Binding ModeNames, Mode=OneWay}" + SelectedValue="{Binding Config.CaptureMode, Mode=TwoWay}" + SelectedValuePath="EnumName"> @@ -213,29 +211,36 @@ + - - - - - + SelectedItem="{Binding Config.HardwareAccelerationConfig.InferenceDevice, Mode=TwoWay}"> + + + + + + @@ -418,7 +423,7 @@ Grid.Column="0" Foreground="{ui:ThemeResource TextFillColorTertiaryBrush}" Text="如果你不知道什么是启动参数请不要填写。" - TextWrapping="Wrap"/> + TextWrapping="Wrap" /> + + + + + + + + + + + + + + + + + 修改后需要重启程序生效，不正确的修改可能会导致程序异常 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 修改后需要重启程序生效。 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 修改后需要重启程序生效。 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/BetterGenshinImpact/View/Pages/View/HardwareAccelerationView.xaml.cs b/BetterGenshinImpact/View/Pages/View/HardwareAccelerationView.xaml.cs new file mode 100644 index 00000000..49e26d7d --- /dev/null +++ b/BetterGenshinImpact/View/Pages/View/HardwareAccelerationView.xaml.cs @@ -0,0 +1,15 @@ +using System.Windows.Controls; +using BetterGenshinImpact.ViewModel.Pages.View; + +namespace BetterGenshinImpact.View.Pages.View; + +public partial class HardwareAccelerationView : UserControl +{ + private HardwareAccelerationViewModel ViewModel { get; } + + public HardwareAccelerationView(HardwareAccelerationViewModel viewModel) + { + DataContext = ViewModel = viewModel; + InitializeComponent(); + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/View/Pages/View/ScriptGroupConfigView.xaml b/BetterGenshinImpact/View/Pages/View/ScriptGroupConfigView.xaml index 5209c3e0..d76e547e 100644 --- a/BetterGenshinImpact/View/Pages/View/ScriptGroupConfigView.xaml +++ b/BetterGenshinImpact/View/Pages/View/ScriptGroupConfigView.xaml @@ -21,7 +21,7 @@ + IsExpanded="True"> @@ -444,8 +444,7 @@ SelectedItem="{Binding PathingConfig.AutoFightConfig.StrategyName, Mode=TwoWay}"> - + @@ -647,68 +646,68 @@ - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/BetterGenshinImpact/ViewModel/MainWindowViewModel.cs b/BetterGenshinImpact/ViewModel/MainWindowViewModel.cs index ca0b0d21..865cad01 100644 --- a/BetterGenshinImpact/ViewModel/MainWindowViewModel.cs +++ b/BetterGenshinImpact/ViewModel/MainWindowViewModel.cs @@ -248,7 +248,7 @@ public partial class MainWindowViewModel : ObservableObject, IViewModel { string gameCultureInfoName = TaskContext.Instance().Config.OtherConfig.GameCultureInfoName; await OcrFactory.ChangeCulture(gameCultureInfoName); - var s = OcrFactory.Paddle.Ocr(new Mat(Global.Absolute(@"Assets\Model\PaddleOCR\test_ocr.png"), ImreadModes.Grayscale)); + var s = OcrFactory.Paddle.Ocr(new Mat(Global.Absolute(@"Assets\Model\PaddleOCR\test_ocr.png"))); Debug.WriteLine("PaddleOcr预热结果:" + s); } catch (Exception e) diff --git a/BetterGenshinImpact/ViewModel/Pages/HomePageViewModel.cs b/BetterGenshinImpact/ViewModel/Pages/HomePageViewModel.cs index ba72dd91..b9da6083 100644 --- a/BetterGenshinImpact/ViewModel/Pages/HomePageViewModel.cs +++ b/BetterGenshinImpact/ViewModel/Pages/HomePageViewModel.cs @@ -30,13 +30,14 @@ using Windows.System; using BetterGenshinImpact.GameTask.AutoFishing; using BetterGenshinImpact.Helpers.Extensions; using BetterGenshinImpact.Model; +using BetterGenshinImpact.View.Pages.View; +using BetterGenshinImpact.ViewModel.Pages.View; using Wpf.Ui.Controls; namespace BetterGenshinImpact.ViewModel.Pages; public partial class HomePageViewModel : ViewModel { - [ObservableProperty] private IEnumerable> _modeNames = EnumExtensions.ToEnumItems(); @@ -66,7 +67,7 @@ public partial class HomePageViewModel : ViewModel private IntPtr _hWnd; [ObservableProperty] - private string[] _inferenceDeviceTypes = BgiSessionOption.InferenceDeviceTypes; + private InferenceDeviceType[] _inferenceDeviceTypes = Enum.GetValues(); public HomePageViewModel(IConfigService configService, TaskTriggerDispatcher taskTriggerDispatcher) { @@ -84,7 +85,9 @@ public partial class HomePageViewModel : ViewModel // DirectML 是在 Windows 10 版本 1903 和 Windows SDK 的相应版本中引入的。 // https://learn.microsoft.com/zh-cn/windows/ai/directml/dml - _inferenceDeviceTypes = _inferenceDeviceTypes.Where(x => x != "GPU_DirectML").ToArray(); + _inferenceDeviceTypes = _inferenceDeviceTypes + .Where(x => x != InferenceDeviceType.GpuDirectMl) + .ToArray(); } WeakReferenceMessenger.Default.Register>(this, (sender, msg) => @@ -157,10 +160,10 @@ public partial class HomePageViewModel : ViewModel } } - [RelayCommand] - private void OnInferenceDeviceTypeDropDownChanged(string value) - { - } + // [RelayCommand] + // private void OnInferenceDeviceTypeDropDownChanged(string value) + // { + // } [RelayCommand] private void OnStartCaptureTest() @@ -437,4 +440,17 @@ public partial class HomePageViewModel : ViewModel win.NavigateToHtml(html); win.ShowDialog(); } + + [RelayCommand] + public void OnOpenHardwareAccelerationSettings() + { + var dialogWindow = new Window + { + Title = "硬件加速设置", + Content = new HardwareAccelerationView(new HardwareAccelerationViewModel()), + SizeToContent = SizeToContent.WidthAndHeight, + WindowStartupLocation = WindowStartupLocation.CenterOwner, + }; + var result = dialogWindow.ShowDialog(); + } } \ No newline at end of file diff --git a/BetterGenshinImpact/ViewModel/Pages/View/HardwareAccelerationViewModel.cs b/BetterGenshinImpact/ViewModel/Pages/View/HardwareAccelerationViewModel.cs new file mode 100644 index 00000000..9b321010 --- /dev/null +++ b/BetterGenshinImpact/ViewModel/Pages/View/HardwareAccelerationViewModel.cs @@ -0,0 +1,32 @@ +using System; +using System.Diagnostics; +using System.Linq; +using BetterGenshinImpact.Core.Config; +using BetterGenshinImpact.Core.Recognition.ONNX; +using BetterGenshinImpact.GameTask; +using CommunityToolkit.Mvvm.Input; + +namespace BetterGenshinImpact.ViewModel.Pages.View; + +using CommunityToolkit.Mvvm.ComponentModel; + +public partial class HardwareAccelerationViewModel : ObservableObject, IViewModel +{ + public HardwareAccelerationConfig Config { get; set; } + public BgiOnnxFactory Status { get; set; } + [ObservableProperty] + private InferenceDeviceType[] _inferenceDeviceTypes = Enum.GetValues(); + [ObservableProperty] + private string _providerTypesText; + public HardwareAccelerationViewModel() + { + Config = TaskContext.Instance().Config.HardwareAccelerationConfig; + Status = BgiOnnxFactory.Instance; + _providerTypesText = string.Join(",", Status.ProviderTypes); + } + [RelayCommand] + public void OnOpenCacheFolder() + { + Process.Start("explorer.exe", Global.Absolute(BgiOnnxModel.ModelCacheRelativePath)); + } +} \ No newline at end of file diff --git a/Fischless.GameCapture/Fischless.GameCapture.csproj b/Fischless.GameCapture/Fischless.GameCapture.csproj index 685c9552..aba49767 100644 --- a/Fischless.GameCapture/Fischless.GameCapture.csproj +++ b/Fischless.GameCapture/Fischless.GameCapture.csproj @@ -14,13 +14,13 @@ - - - - - - - + + + + + + + \ No newline at end of file diff --git a/Fischless.HotkeyCapture/Fischless.HotkeyCapture.csproj b/Fischless.HotkeyCapture/Fischless.HotkeyCapture.csproj index bd52b9cd..bc9a441a 100644 --- a/Fischless.HotkeyCapture/Fischless.HotkeyCapture.csproj +++ b/Fischless.HotkeyCapture/Fischless.HotkeyCapture.csproj @@ -11,7 +11,7 @@ - + diff --git a/Fischless.WindowsInput/Fischless.WindowsInput.csproj b/Fischless.WindowsInput/Fischless.WindowsInput.csproj index 81ef8f50..ac90d079 100644 --- a/Fischless.WindowsInput/Fischless.WindowsInput.csproj +++ b/Fischless.WindowsInput/Fischless.WindowsInput.csproj @@ -10,7 +10,7 @@ - + diff --git a/Test/BetterGenshinImpact.Test/Simple/OcrTest.cs b/Test/BetterGenshinImpact.Test/Simple/OcrTest.cs index be8d039c..0985abe1 100644 --- a/Test/BetterGenshinImpact.Test/Simple/OcrTest.cs +++ b/Test/BetterGenshinImpact.Test/Simple/OcrTest.cs @@ -10,28 +10,14 @@ public class OcrTest { public static void TestYap() { - Mat mat = Cv2.ImRead(@"E:\HuiTask\更好的原神\临时文件\fuben_jueyuan.png", ImreadModes.Grayscale); - var text = TextInferenceFactory.Pick.Inference(PreProcessForInference(mat)); + Mat mat = Cv2.ImRead(@"E:\HuiTask\更好的原神\临时文件\fuben_jueyuan.png"); + var text = TextInferenceFactory.Pick.Inference(TextInferenceFactory.PreProcessForInference(mat)); Debug.WriteLine(text); - Mat mat2 = Cv2.ImRead(@"E:\HuiTask\更好的原神\临时文件\fuben_jueyuan.png", ImreadModes.Grayscale); + Mat mat2 = Cv2.ImRead(@"E:\HuiTask\更好的原神\临时文件\fuben_jueyuan.png"); var text2 = OcrFactory.Paddle.Ocr(mat2); Debug.WriteLine(text2); } - private static Mat PreProcessForInference(Mat mat) - { - // Yap 已经改用灰度图了 https://github.com/Alex-Beng/Yap/commit/c2ad1e7b1442aaf2d80782a032e00876cd1c6c84 - // 二值化 - // Cv2.Threshold(mat, mat, 0, 255, ThresholdTypes.Otsu | ThresholdTypes.Binary); - //Cv2.AdaptiveThreshold(mat, mat, 255, AdaptiveThresholdTypes.GaussianC, ThresholdTypes.Binary, 31, 3); // 效果不错 但是和模型不搭 - //mat = OpenCvCommonHelper.Threshold(mat, Scalar.FromRgb(235, 235, 235), Scalar.FromRgb(255, 255, 255)); // 识别物品不太行 - // 不知道为什么要强制拉伸到 221x32 - mat = ResizeHelper.ResizeTo(mat, 221, 32); - // 填充到 384x32 - var padded = new Mat(new Size(384, 32), MatType.CV_8UC1, Scalar.Black); - padded[new Rect(0, 0, mat.Width, mat.Height)] = mat; - //Cv2.ImWrite(Global.Absolute("padded.png"), padded); - return padded; - } + } diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.GetFishpond.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.GetFishpond.cs index fa52d818..80a9f460 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.GetFishpond.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.GetFishpond.cs @@ -1,15 +1,6 @@ using BetterGenshinImpact.GameTask.AutoFishing; using BehaviourTree; -using System; -using System.Collections.Generic; -using System.Drawing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using BetterGenshinImpact.GameTask.Model.Area; -using BetterGenshinImpact.Core.Config; -using Compunet.YoloV8; -using BetterGenshinImpact.GameTask.AutoFishing.Model; using Microsoft.Extensions.Time.Testing; using OpenCvSharp; diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.ThrowRod.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.ThrowRod.cs index 8fc18c07..bc2c1592 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.ThrowRod.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.ThrowRod.cs @@ -2,13 +2,6 @@ using BetterGenshinImpact.GameTask.AutoFishing; using BetterGenshinImpact.GameTask.Model.Area.Converter; using BetterGenshinImpact.GameTask.Model.Area; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BetterGenshinImpact.Core.Config; -using Compunet.YoloV8; using Microsoft.Extensions.Time.Testing; using OpenCvSharp; using BehaviourTree.Composites; diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.cs index a559b8df..94aa2a58 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoFishingTests/BehavioursTests.cs @@ -1,7 +1,7 @@ using BetterGenshinImpact.Core.Config; using BetterGenshinImpact.Core.Recognition.OCR; +using BetterGenshinImpact.Core.Recognition.ONNX; using BetterGenshinImpact.UnitTest.CoreTests.RecognitionTests.OCRTests; -using Compunet.YoloV8; namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoFishingTests { @@ -9,7 +9,7 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoFishingTests public partial class BehavioursTests { #pragma warning disable CS8618 // 在退出构造函数时，不可为 null 的字段必须包含非 null 值。请考虑添加 "required" 修饰符或声明为可为 null。 - private static YoloV8Predictor predictor; + private static BgiYoloPredictor predictor; #pragma warning restore CS8618 // 在退出构造函数时，不可为 null 的字段必须包含非 null 值。请考虑添加 "required" 修饰符或声明为可为 null。 private readonly PaddleFixture paddle; @@ -18,19 +18,13 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoFishingTests this.paddle = paddle; } - private IOcrService OcrService - { - get - { - return this.paddle.Get(); - } - } + private IOcrService OcrService => paddle.Get(); - private static YoloV8Predictor Predictor + private static BgiYoloPredictor Predictor { get { - return LazyInitializer.EnsureInitialized(ref predictor, () => YoloV8Builder.CreateDefaultBuilder().UseOnnxModel(Global.Absolute(@"Assets\Model\Fish\bgi_fish.onnx")).Build()); + return LazyInitializer.EnsureInitialized(ref predictor,()=>BgiOnnxFactory.Instance.CreateYoloPredictor(BgiOnnxModel.BgiFish)); } } }