diff --git a/BetterGenshinImpact.Test/Simple/AllMap/KeyPointMatchTest.cs b/BetterGenshinImpact.Test/Simple/AllMap/KeyPointMatchTest.cs index b0f946c8..05f120e3 100644 --- a/BetterGenshinImpact.Test/Simple/AllMap/KeyPointMatchTest.cs +++ b/BetterGenshinImpact.Test/Simple/AllMap/KeyPointMatchTest.cs @@ -125,19 +125,19 @@ public class KeyPointMatchTest Stopwatch sw = new Stopwatch(); sw.Start(); - Mat matSrcRet = new Mat(); + using Mat matSrcRet = new Mat(); using Mat matToRet = new Mat(); KeyPoint[] keyPointsSrc, keyPointsTo; using (var surf = OpenCvSharp.XFeatures2D.SURF.Create(threshold, 4, 3, false, true)) { surf.DetectAndCompute(matSrc, null, out keyPointsSrc, matSrcRet); sw.Stop(); - Debug.WriteLine($"大地图kp耗时：{sw.ElapsedMilliseconds}ms."); + Debug.WriteLine($"模板kp耗时：{sw.ElapsedMilliseconds}ms."); sw.Restart(); surf.DetectAndCompute(matTo, null, out keyPointsTo, matToRet); sw.Stop(); - Debug.WriteLine($"模板kp耗时：{sw.ElapsedMilliseconds}ms."); + Debug.WriteLine($"大地图kp耗时：{sw.ElapsedMilliseconds}ms."); sw.Restart(); } diff --git a/BetterGenshinImpact/BetterGenshinImpact.csproj b/BetterGenshinImpact/BetterGenshinImpact.csproj index ec84e86a..37311e0c 100644 --- a/BetterGenshinImpact/BetterGenshinImpact.csproj +++ b/BetterGenshinImpact/BetterGenshinImpact.csproj @@ -27,10 +27,6 @@ - - - - @@ -92,6 +88,27 @@ + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest @@ -535,8 +552,4 @@ - - - - \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/CommonExtension.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/CommonExtension.cs index d5ee9b7e..df3d82c8 100644 --- a/BetterGenshinImpact/Core/Recognition/OpenCv/CommonExtension.cs +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/CommonExtension.cs @@ -1,5 +1,6 @@ using OpenCvSharp; using System; +using System.Collections.Generic; using System.Drawing; using Vanara.PInvoke; using Color = System.Windows.Media.Color; @@ -76,4 +77,14 @@ public static class CommonExtension { return Color.FromArgb(color.A, color.R, color.G, color.B); } + + public static Point2d ToPoint2d(this Point2f p) + { + return new Point2d(p.X, p.Y); + } + + public static List ToPoint2d(this List list) + { + return list.ConvertAll(ToPoint2d); + } } diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/SurfMatcher.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/SurfMatcher.cs new file mode 100644 index 00000000..5ba48468 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/SurfMatcher.cs @@ -0,0 +1,102 @@ +using OpenCvSharp; +using OpenCvSharp.XFeatures2D; +using System.Collections.Generic; +using System; +using System.Diagnostics; +using BetterGenshinImpact.Helpers; + +namespace BetterGenshinImpact.Core.Recognition.OpenCv; + +public class SurfMatcher +{ + private readonly double _threshold; + private readonly Mat _matToRet = new(); + private readonly KeyPoint[] _keyPointsTo; + + public SurfMatcher(Mat matTo, double threshold = 400) + { + _threshold = threshold; + using var surf = SURF.Create(_threshold, 4, 3, false, true); + surf.DetectAndCompute(matTo, null, out _keyPointsTo, _matToRet); + Debug.WriteLine("被匹配的图像生成初始化KeyPoint完成"); + } + + public Point2f[]? Match(Mat matSrc) + { + SpeedTimer speedTimer = new(); + + using var matSrcRet = new Mat(); + KeyPoint[] keyPointsSrc; + using (var surf = SURF.Create(_threshold, 4, 3, false, true)) + { + surf.DetectAndCompute(matSrc, null, out keyPointsSrc, matSrcRet); + speedTimer.Record("模板生成KeyPoint"); + } + + using var flnMatcher = new FlannBasedMatcher(); + var matches = flnMatcher.Match(matSrcRet, _matToRet); + //Finding the Minimum and Maximum Distance + double minDistance = 1000; //Backward approximation + double maxDistance = 0; + for (int i = 0; i < matSrcRet.Rows; i++) + { + double distance = matches[i].Distance; + if (distance > maxDistance) + { + maxDistance = distance; + } + + if (distance < minDistance) + { + minDistance = distance; + } + } + + Debug.WriteLine($"max distance : {maxDistance}"); + Debug.WriteLine($"min distance : {minDistance}"); + + var pointsSrc = new List(); + var pointsDst = new List(); + //Screening better matching points + // var goodMatches = new List(); + for (int i = 0; i < matSrcRet.Rows; i++) + { + double distance = matches[i].Distance; + if (distance < Math.Max(minDistance * 2, 0.02)) + { + pointsSrc.Add(keyPointsSrc[matches[i].QueryIdx].Pt); + pointsDst.Add(_keyPointsTo[matches[i].TrainIdx].Pt); + //Compression of new ones with distances less than ranges DMatch + // goodMatches.Add(matches[i]); + } + } + + speedTimer.Record("FlannMatch"); + + // var outMat = new Mat(); + + // algorithm RANSAC Filter the matched results + var pSrc = pointsSrc.ToPoint2d(); + var pDst = pointsDst.ToPoint2d(); + var outMask = new Mat(); + // If the original matching result is null, Skip the filtering step + if (pSrc.Count > 0 && pDst.Count > 0) + { + var hMat = Cv2.FindHomography(pSrc, pDst, HomographyMethods.Ransac, mask: outMask); + speedTimer.Record("FindHomography"); + + var objCorners = new Point2f[4]; + objCorners[0] = new Point2f(0, 0); + objCorners[1] = new Point2f(0, matSrc.Rows); + objCorners[2] = new Point2f(matSrc.Cols, matSrc.Rows); + objCorners[3] = new Point2f(matSrc.Cols, 0); + + var sceneCorners = Cv2.PerspectiveTransform(objCorners, hMat); + speedTimer.Record("PerspectiveTransform"); + speedTimer.DebugPrint(); + return sceneCorners; + } + speedTimer.DebugPrint(); + return null; + } +} diff --git a/BetterGenshinImpact/GameTask/Common/Map/BigMap.cs b/BetterGenshinImpact/GameTask/Common/Map/BigMap.cs deleted file mode 100644 index 873d7be2..00000000 --- a/BetterGenshinImpact/GameTask/Common/Map/BigMap.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Diagnostics; -using BetterGenshinImpact.Core.Recognition.OpenCv; -using BetterGenshinImpact.Helpers; -using CommunityToolkit.Mvvm.Messaging; -using CommunityToolkit.Mvvm.Messaging.Messages; -using OpenCvSharp; -using Point = OpenCvSharp.Point; -using Size = OpenCvSharp.Size; - -namespace BetterGenshinImpact.GameTask.Common.Map; - -public class BigMap -{ - public static readonly Size TemplateSize = new(240, 135); - - // 对无用部分进行裁剪（左160，上80，下96） - public static readonly Rect TemplateSizeRoi = new Rect(20, 10, TemplateSize.Width - 20, TemplateSize.Height - 22); - - private readonly Mat _mapSrcMat; - - public BigMap() - { - var stream = ResourceHelper.GetStream(@"pack://application:,,,/Assets/Map/map_sd1024.png"); - _mapSrcMat = Mat.FromStream(stream, ImreadModes.Color); - } - - public Point GetMapPosition(Mat captureMat) - { - Cv2.CvtColor(captureMat, captureMat, ColorConversionCodes.BGRA2BGR); - using var tar = new Mat(captureMat.Resize(TemplateSize, 0, 0, InterpolationFlags.Cubic), TemplateSizeRoi); - var p = MatchTemplateHelper.MatchTemplate(_mapSrcMat, tar, TemplateMatchModes.CCoeffNormed, null, 0.2); - Debug.WriteLine($"BigMap Match Template: {p}"); - return p; - } - - public void GetMapPositionAndDraw(Mat captureMat) - { - var p = GetMapPosition(captureMat); - WeakReferenceMessenger.Default.Send(new PropertyChangedMessage(this, "UpdateBigMapRect", new object(), - new System.Windows.Rect(p.X, p.Y, TemplateSizeRoi.Width, TemplateSizeRoi.Height))); - } -} diff --git a/BetterGenshinImpact/GameTask/Common/Map/EntireMap.cs b/BetterGenshinImpact/GameTask/Common/Map/EntireMap.cs new file mode 100644 index 00000000..36839294 --- /dev/null +++ b/BetterGenshinImpact/GameTask/Common/Map/EntireMap.cs @@ -0,0 +1,118 @@ +using BetterGenshinImpact.Core.Config; +using BetterGenshinImpact.Core.Recognition.OpenCv; +using CommunityToolkit.Mvvm.Messaging; +using CommunityToolkit.Mvvm.Messaging.Messages; +using OpenCvSharp; +using System; +using System.Diagnostics; +using Point = OpenCvSharp.Point; +using Size = OpenCvSharp.Size; + +namespace BetterGenshinImpact.GameTask.Common.Map; + +public class EntireMap +{ + // 这个模板缩放大小的计算方式 https://github.com/babalae/better-genshin-impact/issues/318 + public static readonly Size TemplateSize = new(240, 135); + + // 对无用部分进行裁剪（左160，上80，下96） + public static readonly Rect TemplateSizeRoi = new Rect(20, 10, TemplateSize.Width - 20, TemplateSize.Height - 22); + + /// + /// 主要地图缩小1024的模板 + /// + private readonly Mat _mainMap100BlockMat; + + /// + /// 1024区块拼接的主要地图 + /// + private readonly Mat _mainMap1024BlockMat; + + /// + /// 2048城市区块拼接的主要地图 + /// + private readonly Mat _cityMap2048BlockMat; + + private readonly SurfMatcher _surfMatcher; + + public EntireMap() + { + // 大地图模板匹配使用的模板 + _mainMap100BlockMat = new Mat(Global.Absolute(@"Assets\Map\mainMap100Block.png")); + _mainMap1024BlockMat = new Mat(Global.Absolute(@"Assets\Map\mainMap1024Block.png"), ImreadModes.Grayscale); + _cityMap2048BlockMat = new Mat(Global.Absolute(@"Assets\Map\cityMap2048Block.png"), ImreadModes.Grayscale); + _surfMatcher = new SurfMatcher(_mainMap1024BlockMat); + } + + /// + /// 基于模板匹配获取地图位置(100区块，缩小了10.24倍) + /// 当前只支持大地图 + /// + /// 彩色图像 + /// + public Point GetMapPositionByMatchTemplate(Mat captureMat) + { + Cv2.CvtColor(captureMat, captureMat, ColorConversionCodes.BGRA2BGR); + using var tar = new Mat(captureMat.Resize(TemplateSize, 0, 0, InterpolationFlags.Cubic), TemplateSizeRoi); + var p = MatchTemplateHelper.MatchTemplate(_mainMap100BlockMat, tar, TemplateMatchModes.CCoeffNormed, null, 0.2); + Debug.WriteLine($"BigMap Match Template: {p}"); + return p; + } + + public void GetMapPositionAndDrawByMatchTemplate(Mat captureMat) + { + var p = GetMapPositionByMatchTemplate(captureMat); + WeakReferenceMessenger.Default.Send(new PropertyChangedMessage(this, "UpdateBigMapRect", new object(), + new System.Windows.Rect(p.X, p.Y, TemplateSizeRoi.Width, TemplateSizeRoi.Height))); + } + + /// + /// 基于Surf匹配获取地图位置(1024区块) + /// 支持大地图和小地图 + /// + /// 灰度图 + /// + public Rect GetMapPositionBySurf(Mat captureGreyMat) + { + var pArray = _surfMatcher.Match(captureGreyMat); + if (pArray == null || pArray.Length < 4) + { + throw new InvalidOperationException(); + } + + return Cv2.BoundingRect(pArray); + } + + // public static Point GetIntersection(Point2f[] points) + // { + // double a1 = (points[0].Y - points[2].Y) / (double)(points[0].X - points[2].X); + // double b1 = points[0].Y - a1 * points[0].X; + // + // double a2 = (points[1].Y - points[3].Y) / (double)(points[1].X - points[3].X); + // double b2 = points[1].Y - a2 * points[1].X; + // + // if (Math.Abs(a1 - a2) < double.Epsilon) + // { + // // 不相交 + // throw new InvalidOperationException(); + // } + // + // double x = (b2 - b1) / (a1 - a2); + // double y = a1 * x + b1; + // return new Point((int)x, (int)y); + // } + + public void GetMapPositionAndDrawBySurf(Mat captureGreyMat) + { + try + { + var rect = GetMapPositionBySurf(captureGreyMat); + WeakReferenceMessenger.Default.Send(new PropertyChangedMessage(this, "UpdateBigMapRect", new object(), + new System.Windows.Rect(rect.X / 10.24, rect.Y / 10.24, rect.Width / 10.24, rect.Height / 10.24))); + } + catch (Exception) + { + Debug.WriteLine("Surf Match Failed"); + } + } +} diff --git a/BetterGenshinImpact/GameTask/Common/Map/MiniMap.cs b/BetterGenshinImpact/GameTask/Common/Map/MiniMap.cs deleted file mode 100644 index c7198ea1..00000000 --- a/BetterGenshinImpact/GameTask/Common/Map/MiniMap.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace BetterGenshinImpact.GameTask.Common.Map; - -public class MiniMap -{ -} diff --git a/BetterGenshinImpact/GameTask/Placeholder/PlaceholderTrigger.cs b/BetterGenshinImpact/GameTask/Placeholder/PlaceholderTrigger.cs index 3a372646..94625005 100644 --- a/BetterGenshinImpact/GameTask/Placeholder/PlaceholderTrigger.cs +++ b/BetterGenshinImpact/GameTask/Placeholder/PlaceholderTrigger.cs @@ -1,5 +1,4 @@ -using BetterGenshinImpact.GameTask.Common.Map; -using BetterGenshinImpact.View.Drawable; +using BetterGenshinImpact.View.Drawable; using OpenCvSharp; using System; using System.Collections.Generic; @@ -27,7 +26,7 @@ public class TestTrigger : ITaskTrigger // private readonly YoloV8 _predictor = new(Global.Absolute("Assets\\Model\\Domain\\bgi_tree.onnx")); - private readonly BigMap _bigMap = new(); + // private readonly EntireMap _bigMap = new(); public TestTrigger() { @@ -93,7 +92,7 @@ public class TestTrigger : ITaskTrigger // Debug.WriteLine("没找到"); //} - _bigMap.GetMapPositionAndDraw(content.CaptureRectArea.SrcMat); + // _bigMap.GetMapPositionAndDrawBySurf(content.CaptureRectArea.SrcGreyMat); } // private void Detect(CaptureContent content) diff --git a/BetterGenshinImpact/View/Windows/MapViewer.xaml b/BetterGenshinImpact/View/Windows/MapViewer.xaml index a52a8ae8..873f9240 100644 --- a/BetterGenshinImpact/View/Windows/MapViewer.xaml +++ b/BetterGenshinImpact/View/Windows/MapViewer.xaml @@ -25,7 +25,7 @@ - + - + \ No newline at end of file diff --git a/BetterGenshinImpact/ViewModel/Windows/MapViewerViewModel.cs b/BetterGenshinImpact/ViewModel/Windows/MapViewerViewModel.cs index a19dea23..e23433d3 100644 --- a/BetterGenshinImpact/ViewModel/Windows/MapViewerViewModel.cs +++ b/BetterGenshinImpact/ViewModel/Windows/MapViewerViewModel.cs @@ -1,4 +1,6 @@ -using System.Windows; +using System; +using System.Windows; +using BetterGenshinImpact.Core.Config; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Messaging; using CommunityToolkit.Mvvm.Messaging.Messages; @@ -10,6 +12,9 @@ public partial class MapViewerViewModel : ObservableObject [ObservableProperty] private Rect _bigMapRect = new(0, 0, 0, 0); + [ObservableProperty] + private string _mapPath = Global.Absolute(@"Assets\Map\mainMap100Block.png"); + public MapViewerViewModel() { WeakReferenceMessenger.Default.Register>(this, (sender, msg) =>