diff --git a/BetterGenshinImpact.Test/MainWindow.xaml.cs b/BetterGenshinImpact.Test/MainWindow.xaml.cs index d0d91d04..41b8851a 100644 --- a/BetterGenshinImpact.Test/MainWindow.xaml.cs +++ b/BetterGenshinImpact.Test/MainWindow.xaml.cs @@ -49,7 +49,8 @@ public partial class MainWindow : Window { // KeyPointMatchTest.Test(); // EntireMapTest.Test(); - EntireMapTest.Storage(); + // EntireMapTest.Storage(); + BigMapMatchTest.Test(); } private void MapDrawTeleportPoint(object sender, RoutedEventArgs e) diff --git a/BetterGenshinImpact.Test/Simple/AllMap/BigMapMatchTest.cs b/BetterGenshinImpact.Test/Simple/AllMap/BigMapMatchTest.cs new file mode 100644 index 00000000..86eaf744 --- /dev/null +++ b/BetterGenshinImpact.Test/Simple/AllMap/BigMapMatchTest.cs @@ -0,0 +1,46 @@ +using BetterGenshinImpact.Core.Recognition.OpenCv.FeatureMatch; +using BetterGenshinImpact.GameTask.Common.Element.Assets; +using BetterGenshinImpact.Helpers; +using OpenCvSharp; +using OpenCvSharp.Detail; +using System.Diagnostics; +using System.Windows.Forms; +using BetterGenshinImpact.Core.Recognition.OpenCv; + +namespace BetterGenshinImpact.Test.Simple.AllMap; + +public class BigMapMatchTest +{ + public static void Test() + { + SpeedTimer speedTimer = new(); + // var mainMap100BlockMat = new Mat(@"D:\HuiPrograming\Projects\CSharp\MiHoYo\BetterGenshinImpact\BetterGenshinImpact\Assets\Map\mainMap100Block.png", ImreadModes.Grayscale); + + var map2048 = MapAssets.Instance.MainMap2048BlockMat.Value; + var mainMap100BlockMat = ResizeHelper.Resize(map2048, 1d / (4 * 2)); + Cv2.ImWrite(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\mainMap128Block.png", mainMap100BlockMat); + + var surfMatcher = new FeatureMatcher(mainMap100BlockMat); + var queryMat = new Mat(@"E:\HuiTask\更好的原神\地图匹配\比较\Clip_20240321_000329.png", ImreadModes.Grayscale); + + speedTimer.Record("初始化特征"); + + queryMat = ResizeHelper.Resize(queryMat, 1d / 4); + Cv2.ImShow("queryMat", queryMat); + + var pArray = surfMatcher.Match(queryMat); + speedTimer.Record("匹配1"); + if (pArray != null) + { + var rect = Cv2.BoundingRect(pArray); + Debug.WriteLine($"Matched rect 1: {rect}"); + Cv2.Rectangle(mainMap100BlockMat, rect, Scalar.Red, 2); + Cv2.ImShow(@"b1", mainMap100BlockMat); + } + else + { + Debug.WriteLine("No match 1"); + } + speedTimer.DebugPrint(); + } +} diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/FeatureMatch/FeatureMatcher.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/FeatureMatch/FeatureMatcher.cs index 1b074a1d..8cf9d519 100644 --- a/BetterGenshinImpact/Core/Recognition/OpenCv/FeatureMatch/FeatureMatcher.cs +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/FeatureMatch/FeatureMatcher.cs @@ -14,7 +14,10 @@ public class FeatureMatcher { private readonly double _threshold = 100; // SURF 100 private readonly Feature2D _feature2D; - private readonly Mat _trainMat; // 大图本身 + + // private readonly Mat _trainMat; // 大图本身 + private readonly Size _trainMatSize; // 大图大小 + private readonly Mat _trainRet = new(); // 大图特征描述子 private readonly KeyPoint[] _trainKeyPoints; @@ -23,10 +26,19 @@ public class FeatureMatcher private readonly int _splitCol = MapCoordinate.GameMapCols * 2; // 特征点拆分列数 private KeyPointFeatureBlock? _lastMergedBlock; // 上次合并的特征块 + /// + /// 从图像 or 特征点加载 + /// 大图不建议使用此构造函数加载，速度很慢 + /// + /// + /// + /// + /// + /// public FeatureMatcher(Mat trainMat, FeatureStorage? featureStorage = null, Feature2DType type = Feature2DType.SIFT, double threshold = 100) { _threshold = threshold; - _trainMat = trainMat; + _trainMatSize = trainMat.Size(); if (Feature2DType.SURF == type) { _feature2D = SURF.Create(_threshold, 4, 3, false, true); @@ -54,11 +66,51 @@ public class FeatureMatcher _trainRet = featureStorage.LoadDescMat() ?? throw new Exception("加载特征描述矩阵失败"); } } + else + { + _feature2D.DetectAndCompute(trainMat, null, out _trainKeyPoints, _trainRet); + } Debug.WriteLine("被匹配的图像生成初始化KeyPoint完成"); Stopwatch sw = new(); sw.Start(); - _blocks = KeyPointFeatureBlockHelper.SplitFeatures(_trainMat, _splitRow, _splitCol, _trainKeyPoints, _trainRet); + _blocks = KeyPointFeatureBlockHelper.SplitFeatures(_trainMatSize, _splitRow, _splitCol, _trainKeyPoints, _trainRet); + sw.Stop(); + Debug.WriteLine($"切割特征点耗时: {sw.ElapsedMilliseconds}ms"); + } + + /// + /// 直接从特征点加载 + /// + /// + /// + /// + /// + /// + public FeatureMatcher(Size trainMatSize, FeatureStorage featureStorage, Feature2DType type = Feature2DType.SIFT, double threshold = 100) + { + _threshold = threshold; + _trainMatSize = trainMatSize; + if (Feature2DType.SURF == type) + { + _feature2D = SURF.Create(_threshold, 4, 3, false, true); + } + else + { + _feature2D = SIFT.Create(); + } + + featureStorage.TypeName = type.ToString(); + Debug.WriteLine("尝试从磁盘加载特征点"); + var kpFromDisk = featureStorage.LoadKeyPointArray(); + + _trainKeyPoints = kpFromDisk ?? throw new Exception("特征点不存在"); + _trainRet = featureStorage.LoadDescMat() ?? throw new Exception("加载特征描述矩阵失败"); + + Debug.WriteLine("被匹配的图像生成初始化KeyPoint完成"); + Stopwatch sw = new(); + sw.Start(); + _blocks = KeyPointFeatureBlockHelper.SplitFeatures(_trainMatSize, _splitRow, _splitCol, _trainKeyPoints, _trainRet); sw.Stop(); Debug.WriteLine($"切割特征点耗时: {sw.ElapsedMilliseconds}ms"); } @@ -83,7 +135,7 @@ public class FeatureMatcher /// public Point2f[]? Match(Mat queryMat, int prevX, int prevY, Mat? queryMatMask = null) { - var (cellRow, cellCol) = KeyPointFeatureBlockHelper.GetCellIndex(_trainMat, _splitRow, _splitCol, prevX, prevY); + var (cellRow, cellCol) = KeyPointFeatureBlockHelper.GetCellIndex(_trainMatSize, _splitRow, _splitCol, prevX, prevY); Debug.WriteLine($"当前坐标({prevX},{prevY})在特征块({cellRow},{cellCol})中"); if (_lastMergedBlock == null || _lastMergedBlock.MergedCenterCellRow != cellRow || _lastMergedBlock.MergedCenterCellCol != cellCol) { diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/FeatureMatch/KeyPointFeatureBlockHelper.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/FeatureMatch/KeyPointFeatureBlockHelper.cs index d4af76d3..55c01e2f 100644 --- a/BetterGenshinImpact/Core/Recognition/OpenCv/FeatureMatch/KeyPointFeatureBlockHelper.cs +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/FeatureMatch/KeyPointFeatureBlockHelper.cs @@ -9,7 +9,7 @@ namespace BetterGenshinImpact.Core.Recognition.OpenCv.FeatureMatch; public class KeyPointFeatureBlockHelper { - public static KeyPointFeatureBlock[][] SplitFeatures(Mat originalImage, int rows, int cols, KeyPoint[] keyPoints, Mat matches) + public static KeyPointFeatureBlock[][] SplitFeatures(Size originalImage, int rows, int cols, KeyPoint[] keyPoints, Mat matches) { var matchesCols = matches.Cols; // SURF 64 SIFT 128 // Calculate grid size @@ -68,7 +68,7 @@ public class KeyPointFeatureBlockHelper return splitKeyPoints; } - public static (int, int) GetCellIndex(Mat originalImage, int rows, int cols, int x, int y) + public static (int, int) GetCellIndex(Size originalImage, int rows, int cols, int x, int y) { // Calculate grid size int cellWidth = originalImage.Width / cols; diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/ResizeHelper.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/ResizeHelper.cs index 028d46c1..712db3c7 100644 --- a/BetterGenshinImpact/Core/Recognition/OpenCv/ResizeHelper.cs +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/ResizeHelper.cs @@ -10,22 +10,23 @@ public class ResizeHelper /// /// /// + /// /// - public static Mat Resize(Mat src, double scale) + public static Mat Resize(Mat src, double scale, InterpolationFlags interpolation = InterpolationFlags.Linear) { if (Math.Abs(scale - 1) > 0.00001) { - return Resize(src, scale, scale); + return Resize(src, scale, scale, interpolation); } return src; } - public static Mat Resize(Mat src, double widthScale, double heightScale) + public static Mat Resize(Mat src, double widthScale, double heightScale, InterpolationFlags interpolation = InterpolationFlags.Linear) { if (Math.Abs(widthScale - 1) > 0.00001 || Math.Abs(heightScale - 1) > 0.00001) { var dst = new Mat(); - Cv2.Resize(src, dst, new Size(src.Width * widthScale, src.Height * heightScale)); + Cv2.Resize(src, dst, new Size(src.Width * widthScale, src.Height * heightScale), 0, 0, interpolation); return dst; } diff --git a/BetterGenshinImpact/GameTask/AutoTrackPath/AutoTrackPathTask.cs b/BetterGenshinImpact/GameTask/AutoTrackPath/AutoTrackPathTask.cs index ab100091..067ee45d 100644 --- a/BetterGenshinImpact/GameTask/AutoTrackPath/AutoTrackPathTask.cs +++ b/BetterGenshinImpact/GameTask/AutoTrackPath/AutoTrackPathTask.cs @@ -66,7 +66,7 @@ public class AutoTrackPathTask var json = File.ReadAllText(Global.Absolute(@"GameTask\AutoTrackPath\Assets\tp.json")); _tpPositions = JsonSerializer.Deserialize>(json, ConfigService.JsonOptions) ?? throw new Exception("tp.json deserialize failed"); - var wayJson = File.ReadAllText(Global.Absolute(@"log\way\yl3.json")); + var wayJson = File.ReadAllText(Global.Absolute(@"log\way\way2.json")); _way = JsonSerializer.Deserialize(wayJson, ConfigService.JsonOptions) ?? throw new Exception("way json deserialize failed"); } @@ -526,9 +526,10 @@ public class AutoTrackPathTask using var mapScaleButtonRa = ra.Find(QuickTeleportAssets.Instance.MapScaleButtonRo); if (mapScaleButtonRa.IsExist()) { - var rect = EntireMap.Instance.GetBigMapPositionByFeatureMatch(ra.SrcGreyMat); + var rect = BigMap.Instance.GetBigMapPositionByFeatureMatch(ra.SrcGreyMat); Debug.WriteLine("识别大地图在全地图位置矩形：" + rect); - return MapCoordinate.Main2048ToGame(rect); + const int s = 4 * 2; // 相对1024做4倍缩放 + return MapCoordinate.Main2048ToGame(new Rect(rect.X * s, rect.Y * s, rect.Width * s, rect.Height * s)); } else { diff --git a/BetterGenshinImpact/GameTask/Common/Element/Assets/MapAssets.cs b/BetterGenshinImpact/GameTask/Common/Element/Assets/MapAssets.cs index c18c32df..cda1e424 100644 --- a/BetterGenshinImpact/GameTask/Common/Element/Assets/MapAssets.cs +++ b/BetterGenshinImpact/GameTask/Common/Element/Assets/MapAssets.cs @@ -9,10 +9,13 @@ public class MapAssets : BaseAssets { public Lazy MainMap100BlockMat { get; } = new(() => new Mat(Global.Absolute(@"Assets\Map\mainMap100Block.png"))); - public Lazy MainMap1024BlockMat { get; } = new(() => new Mat(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\mainMap1024Block.png", ImreadModes.Grayscale)); + // public Lazy MainMap1024BlockMat { get; } = new(() => new Mat(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\mainMap1024Block.png", ImreadModes.Grayscale)); public Lazy MainMap2048BlockMat { get; } = new(() => new Mat(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\mainMap2048Block.png", ImreadModes.Grayscale)); + public Lazy MainMap128BlockMat { get; } = new(() => new Mat(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\mainMap128Block.png", ImreadModes.Grayscale)); + public Lazy MainMap256BlockMat { get; } = new(() => new Mat(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\mainMap256Block.png", ImreadModes.Grayscale)); + // 每个地区点击后处于的中心位置 // 2048 区块下，存在传送点的最大面积，识别结果比这个大的话，需要点击放大 diff --git a/BetterGenshinImpact/GameTask/Common/Map/BigMap.cs b/BetterGenshinImpact/GameTask/Common/Map/BigMap.cs new file mode 100644 index 00000000..675cf935 --- /dev/null +++ b/BetterGenshinImpact/GameTask/Common/Map/BigMap.cs @@ -0,0 +1,44 @@ +using System; +using System.Diagnostics; +using BetterGenshinImpact.Core.Recognition.OpenCv; +using BetterGenshinImpact.Core.Recognition.OpenCv.FeatureMatch; +using BetterGenshinImpact.GameTask.Common.Element.Assets; +using BetterGenshinImpact.Model; +using OpenCvSharp; + +namespace BetterGenshinImpact.GameTask.Common.Map; + +/// +/// 专门用于大地图的识别 +/// 图像缩小了8倍 +/// +public class BigMap : Singleton +{ + // 直接从图像加载特征点 + private readonly FeatureMatcher _featureMatcher = new(MapAssets.Instance.MainMap256BlockMat.Value, new FeatureStorage("mainMap256Block")); + + /// + /// 基于特征匹配获取地图位置 全部匹配 + /// + /// 传入的大地图图像会缩小8倍 + /// + public Rect GetBigMapPositionByFeatureMatch(Mat greyMat) + { + try + { + greyMat = ResizeHelper.Resize(greyMat, 1d / 4); + + var pArray = _featureMatcher.Match(greyMat); + if (pArray == null || pArray.Length < 4) + { + throw new InvalidOperationException(); + } + return Cv2.BoundingRect(pArray); + } + catch + { + Debug.WriteLine("Feature Match Failed"); + return Rect.Empty; + } + } +} diff --git a/BetterGenshinImpact/GameTask/Common/Map/EntireMap.cs b/BetterGenshinImpact/GameTask/Common/Map/EntireMap.cs index daeade1d..2439a9d9 100644 --- a/BetterGenshinImpact/GameTask/Common/Map/EntireMap.cs +++ b/BetterGenshinImpact/GameTask/Common/Map/EntireMap.cs @@ -49,7 +49,8 @@ public class EntireMap : Singleton // Mat grey = new(); // Cv2.CvtColor(_mainMap100BlockMat, grey, ColorConversionCodes.BGR2GRAY); // _featureMatcher = new FeatureMatcher(MapAssets.Instance.MainMap1024BlockMat.Value, new FeatureStorage("mainMap1024Block")); - _featureMatcher = new FeatureMatcher(MapAssets.Instance.MainMap2048BlockMat.Value, new FeatureStorage("mainMap2048Block")); + // 只从特征点加载 + _featureMatcher = new FeatureMatcher(new Size(28672, 26624), new FeatureStorage("mainMap2048Block")); } /// diff --git a/BetterGenshinImpact/GameTask/Placeholder/PlaceholderTrigger.cs b/BetterGenshinImpact/GameTask/Placeholder/PlaceholderTrigger.cs index 7a883b20..b07c172a 100644 --- a/BetterGenshinImpact/GameTask/Placeholder/PlaceholderTrigger.cs +++ b/BetterGenshinImpact/GameTask/Placeholder/PlaceholderTrigger.cs @@ -9,6 +9,8 @@ using System.Linq; using BetterGenshinImpact.Core.Config; using BetterGenshinImpact.GameTask.Common.BgiVision; using BetterGenshinImpact.GameTask.Common.Element.Assets; +using CommunityToolkit.Mvvm.Messaging; +using CommunityToolkit.Mvvm.Messaging.Messages; using Point = OpenCvSharp.Point; using Size = OpenCvSharp.Size; @@ -96,25 +98,29 @@ public class TestTrigger : ITaskTrigger //} // 小地图匹配测试 - var tar = ElementAssets.Instance.PaimonMenuRo.TemplateImageGreyMat!; - var p = MatchTemplateHelper.MatchTemplate(content.CaptureRectArea.SrcGreyMat, tar, TemplateMatchModes.CCoeffNormed, null, 0.9); - if (p.X == 0 || p.Y == 0) - { - return; - } - - var miniMapMat = new Mat(content.CaptureRectArea.SrcGreyMat, new Rect(p.X + 24, p.Y - 15, 210, 210)); - var mask = new Mat(new Size(miniMapMat.Width, miniMapMat.Height), MatType.CV_8UC1, Scalar.Black); - Cv2.Circle(mask, new Point(miniMapMat.Width / 2, miniMapMat.Height / 2), 90, Scalar.White, -1); - var res = new Mat(); - Cv2.BitwiseAnd(miniMapMat, miniMapMat, res, mask); - EntireMap.Instance.GetMapPositionAndDrawByFeatureMatch(res); - Cv2.ImWrite(Global.Absolute(@"log\minimap.png"), res); + // var tar = ElementAssets.Instance.PaimonMenuRo.TemplateImageGreyMat!; + // var p = MatchTemplateHelper.MatchTemplate(content.CaptureRectArea.SrcGreyMat, tar, TemplateMatchModes.CCoeffNormed, null, 0.9); + // if (p.X == 0 || p.Y == 0) + // { + // return; + // } + // + // var miniMapMat = new Mat(content.CaptureRectArea.SrcGreyMat, new Rect(p.X + 24, p.Y - 15, 210, 210)); + // var mask = new Mat(new Size(miniMapMat.Width, miniMapMat.Height), MatType.CV_8UC1, Scalar.Black); + // Cv2.Circle(mask, new Point(miniMapMat.Width / 2, miniMapMat.Height / 2), 90, Scalar.White, -1); + // var res = new Mat(); + // Cv2.BitwiseAnd(miniMapMat, miniMapMat, res, mask); + // EntireMap.Instance.GetMapPositionAndDrawByFeatureMatch(res); + // Cv2.ImWrite(Global.Absolute(@"log\minimap.png"), res); // 大地图测试 - // var mat = content.CaptureRectArea.SrcGreyMat; - // // mat = mat.Resize(new Size(240, 135)); - // EntireMap.Instance.GetMapPositionAndDrawByFeatureMatch(mat); + var mat = content.CaptureRectArea.SrcGreyMat; + // mat = mat.Resize(new Size(240, 135)); + Rect rect = BigMap.Instance.GetBigMapPositionByFeatureMatch(mat); + + var s = 2.56; + WeakReferenceMessenger.Default.Send(new PropertyChangedMessage(this, "UpdateBigMapRect", new object(), + new System.Windows.Rect(rect.X / s, rect.Y / s, rect.Width / s, rect.Height / s))); // Bv.BigMapIsUnderground(content.CaptureRectArea); }