From cbaf1afdfc3d191bb192527b646408d68ff38561 Mon Sep 17 00:00:00 2001 From: huiyadanli Date: Mon, 2 Oct 2023 15:07:28 +0800 Subject: [PATCH] refactor: Identification Method Replacement and Packaging --- .../Core/Config/ApplicationConfiguration.cs | 2 +- .../Core/Recognition/OpenCv/Easy.cs | 4 +- .../Recognition/OpenCv/MatchTemplateHelper.cs | 181 +++-------------- .../OpenCv/OldMatchTemplateHelper.cs | 185 ++++++++++++++++++ .../Recognition/OpenCv/OpenCvCommonHelper.cs | 16 ++ .../Core/Recognition/RecognitionObject.cs | 46 ++++- .../Core/Simulator/MouseEventSimulator.cs | 1 + .../AutoFishing/AutoFishingTrigger.cs | 2 +- .../GameTask/AutoPick/AutoPickTrigger.cs | 2 +- .../AutoSkip/Assets/AutoSkipAssets.cs | 50 +++-- .../GameTask/AutoSkip/AutoSkipTrigger.cs | 95 +++++---- .../GameTask/CaptureContent.cs | 6 +- .../GameTask/GameTaskManager.cs | 44 ++++- .../GameTask/Model/RectArea.cs | 101 ++++++++-- .../GameTask/Model/SystemInfo.cs | 34 ++++ BetterGenshinImpact/GameTask/SystemControl.cs | 4 +- BetterGenshinImpact/GameTask/TaskContext.cs | 5 +- .../GameTask/TaskDispatcher.cs | 5 +- .../Helpers/Extensions/BitmapExtension.cs | 13 +- .../Simulator => Helpers}/PrimaryScreen.cs | 7 +- 20 files changed, 557 insertions(+), 246 deletions(-) create mode 100644 BetterGenshinImpact/Core/Recognition/OpenCv/OldMatchTemplateHelper.cs create mode 100644 BetterGenshinImpact/GameTask/Model/SystemInfo.cs rename BetterGenshinImpact/{Core/Simulator => Helpers}/PrimaryScreen.cs (94%) diff --git a/BetterGenshinImpact/Core/Config/ApplicationConfiguration.cs b/BetterGenshinImpact/Core/Config/ApplicationConfiguration.cs index b9f5b9e5..36eec6e6 100644 --- a/BetterGenshinImpact/Core/Config/ApplicationConfiguration.cs +++ b/BetterGenshinImpact/Core/Config/ApplicationConfiguration.cs @@ -24,7 +24,7 @@ namespace BetterGenshinImpact.Core.Config /// /// 触发器触发频率(ms) /// - public int FrameInterval { get; set; } = 50; + public int TriggerInterval { get; set; } = 100; /// /// 遮罩窗口配置 diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/Easy.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/Easy.cs index 277f2faa..5c36b916 100644 --- a/BetterGenshinImpact/Core/Recognition/OpenCv/Easy.cs +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/Easy.cs @@ -23,7 +23,7 @@ namespace BetterGenshinImpact.Core.Recognition.OpenCv public Point Click(Mat targetMat, double threshold = 0.8, int intervalMillisecond = 300) { - Point p = MatchTemplateHelper.FindSingleTarget(SrcMat, targetMat, threshold); + Point p = OldMatchTemplateHelper.FindSingleTarget(SrcMat, targetMat, threshold); if (p.X > 0 && p.Y > 0) { //VisionContext.Instance().DrawContent.PutRect("ClickMatch", new System.Windows.Rect( @@ -37,7 +37,7 @@ namespace BetterGenshinImpact.Core.Recognition.OpenCv public Point Exist(Mat targetMat, double threshold = 0.8) { - return MatchTemplateHelper.FindSingleTarget(SrcMat, targetMat, threshold); + return OldMatchTemplateHelper.FindSingleTarget(SrcMat, targetMat, threshold); } } } \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/MatchTemplateHelper.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/MatchTemplateHelper.cs index 17747360..9c343654 100644 --- a/BetterGenshinImpact/Core/Recognition/OpenCv/MatchTemplateHelper.cs +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/MatchTemplateHelper.cs @@ -18,168 +18,43 @@ namespace BetterGenshinImpact.Core.Recognition.OpenCv public static double WidthScale = 1; public static double HeightScale = 1; - - public static Point FindSingleTarget(Bitmap imgSrc, Bitmap imgSub, double threshold = 0.8) + /// + /// 模板匹配 + /// + /// + /// + /// + /// + /// + /// 左上角的标点 + public static Point MatchTemplate(Mat srcMat, Mat dstMat, TemplateMatchModes matchMode, Mat? maskMat = null, double threshold = 0.8) { - Mat? srcMat = null; - Mat? dstMat = null; try { - srcMat = imgSrc.ToMat(); - dstMat = imgSub.ToMat(); - return FindSingleTarget(srcMat, dstMat, threshold); + var result = new Mat(); + if (maskMat == null) + { + Cv2.MatchTemplate(srcMat, dstMat, result, matchMode); + } + else + { + Cv2.MatchTemplate(srcMat, dstMat, result, matchMode, maskMat); + } + + Cv2.MinMaxLoc(result, out _, out var maxValue, out _, out var point); + + if (maxValue >= threshold) + { + return point; + } + + return new Point(); } catch (Exception ex) { _logger.LogError(ex.ToString()); return new Point(); } - finally - { - srcMat?.Dispose(); - dstMat?.Dispose(); - } - } - - public static Point FindSingleTarget(Mat srcMat, Mat dstMat, double threshold = 0.8) - { - Point p = new Point(); - - OutputArray? outArray = null; - try - { - dstMat = ResizeHelper.Resize(dstMat, WidthScale); - - outArray = OutputArray.Create(srcMat); - Cv2.MatchTemplate(srcMat, dstMat, outArray, TemplateMatchModes.CCoeffNormed); - double minValue, maxValue; - Point location, point; - Cv2.MinMaxLoc(InputArray.Create(outArray.GetMat()), out minValue, out maxValue, - out location, out point); - - if (maxValue >= threshold) - { - p = new Point(point.X + dstMat.Width / 2, point.Y + dstMat.Height / 2); - //if (VisionContext.Instance().Drawable) - //{ - //VisionContext.Instance().DrawContent.PutRect("", new System.Windows.Rect(point.X, point.Y, dstMat.Width, dstMat.Height)); - //VisionContext.Instance().DrawContent.TextList - // .Add(new Tuple(new System.Windows.Point(point.X, point.Y - 10), maxValue.ToString("0.00"))); - //} - } - - return p; - } - catch (Exception ex) - { - _logger.LogError(ex.ToString()); - return p; - } - finally - { - outArray?.Dispose(); - } - } - - public static List FindMultiTarget(Mat srcMat, Mat dstMat, string title, out Mat resMat, - double threshold = 0.8, int findTargetCount = 8) - { - List pointList = new List(); - resMat = srcMat.Clone(); - try - { - dstMat = ResizeHelper.Resize(dstMat, WidthScale); - - Mat matchResult = new Mat(); - Cv2.MatchTemplate(srcMat, dstMat, matchResult, TemplateMatchModes.CCoeffNormed); - - double minValue = 0; - double maxValue = 0; - Point minLoc = new(); - - //寻找最几个最值的位置 - Mat mask = new Mat(matchResult.Height, matchResult.Width, MatType.CV_8UC1, Scalar.White); - Mat maskSub = new Mat(matchResult.Height, matchResult.Width, MatType.CV_8UC1, Scalar.Black); - var point = new OpenCvSharp.Point(0, 0); - for (int i = 0; i < findTargetCount; i++) - { - Cv2.MinMaxLoc(matchResult, out minValue, out maxValue, out minLoc, out point, mask); - Rect maskRect = new Rect(point.X - dstMat.Width / 2, point.Y - dstMat.Height / 2, dstMat.Width, - dstMat.Height); - maskSub.Rectangle(maskRect, Scalar.White, -1); - mask -= maskSub; - if (maxValue >= threshold) - { - pointList.Add(new Point(point.X + dstMat.Width / 2, point.Y + dstMat.Height / 2)); - - //if (VisionContext.Instance().Drawable) - //{ - // VisionContext.Instance().DrawContent.RectList - // .Add(new System.Windows.Rect(point.X, point.Y, dstMat.Width, dstMat.Height)); - // VisionContext.Instance().DrawContent.TextList - // .Add(new Tuple(new System.Windows.Point(point.X, point.Y - 10), maxValue.ToString("0.00"))); - //} - //if (IsDebug) - //{ - // VisionContext.Instance().Log - // ?.LogInformation(title + " " + maxValue.ToString("0.000") + " " + point); - // Cv2.Rectangle(resMat, point, - // new OpenCvSharp.Point(point.X + dstMat.Width, point.Y + dstMat.Height), - // Scalar.Red, 2); - // Cv2.PutText(resMat, title + " " + maxValue.ToString("0.00"), - // new OpenCvSharp.Point(point.X, point.Y - 10), - // HersheyFonts.HersheySimplex, 0.5, Scalar.Red); - //} - } - else - { - break; - } - } - - return pointList; - } - catch (Exception ex) - { - _logger.LogError(ex.ToString()); - return pointList; - } - finally - { - srcMat?.Dispose(); - dstMat?.Dispose(); - } - } - - - public static Dictionary> FindMultiPicFromOneImage(Bitmap imgSrc, - Dictionary imgSubDictionary, double threshold = 0.8) - { - Dictionary> dictionary = new Dictionary>(); - Mat srcMat = imgSrc.ToMat(); - Mat resMat; - - foreach (KeyValuePair kvp in imgSubDictionary) - { - dictionary.Add(kvp.Key, FindMultiTarget(srcMat, kvp.Value.ToMat(), kvp.Key, out resMat, threshold)); - srcMat = resMat.Clone(); - } - - return dictionary; - } - - public static Dictionary> FindMultiPicFromOneImage(Mat srcMat, - Dictionary imgSubDictionary, double threshold = 0.8) - { - Dictionary> dictionary = new Dictionary>(); - Mat resMat; - foreach (KeyValuePair kvp in imgSubDictionary) - { - dictionary.Add(kvp.Key, FindMultiTarget(srcMat, kvp.Value.ToMat(), kvp.Key, out resMat, threshold)); - srcMat = resMat.Clone(); - } - - return dictionary; } } } \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/OldMatchTemplateHelper.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/OldMatchTemplateHelper.cs new file mode 100644 index 00000000..89da7145 --- /dev/null +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/OldMatchTemplateHelper.cs @@ -0,0 +1,185 @@ +using OpenCvSharp; +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Point = OpenCvSharp.Point; +using BetterGenshinImpact.GameTask.AutoSkip; + +namespace BetterGenshinImpact.Core.Recognition.OpenCv +{ + public class OldMatchTemplateHelper + { + private static readonly ILogger _logger = App.GetLogger(); + + public static double WidthScale = 1; + public static double HeightScale = 1; + + + public static Point FindSingleTarget(Bitmap imgSrc, Bitmap imgSub, double threshold = 0.8) + { + Mat? srcMat = null; + Mat? dstMat = null; + try + { + srcMat = imgSrc.ToMat(); + dstMat = imgSub.ToMat(); + return FindSingleTarget(srcMat, dstMat, threshold); + } + catch (Exception ex) + { + _logger.LogError(ex.ToString()); + return new Point(); + } + finally + { + srcMat?.Dispose(); + dstMat?.Dispose(); + } + } + + public static Point FindSingleTarget(Mat srcMat, Mat dstMat, double threshold = 0.8) + { + Point p = new Point(); + + OutputArray? outArray = null; + try + { + dstMat = ResizeHelper.Resize(dstMat, WidthScale); + + outArray = OutputArray.Create(srcMat); + Cv2.MatchTemplate(srcMat, dstMat, outArray, TemplateMatchModes.CCoeffNormed); + double minValue, maxValue; + Point location, point; + Cv2.MinMaxLoc(InputArray.Create(outArray.GetMat()), out minValue, out maxValue, + out location, out point); + + if (maxValue >= threshold) + { + p = new Point(point.X + dstMat.Width / 2, point.Y + dstMat.Height / 2); + //if (VisionContext.Instance().Drawable) + //{ + //VisionContext.Instance().DrawContent.PutRect("", new System.Windows.Rect(point.X, point.Y, dstMat.Width, dstMat.Height)); + //VisionContext.Instance().DrawContent.TextList + // .Add(new Tuple(new System.Windows.Point(point.X, point.Y - 10), maxValue.ToString("0.00"))); + //} + } + + return p; + } + catch (Exception ex) + { + _logger.LogError(ex.ToString()); + return p; + } + finally + { + outArray?.Dispose(); + } + } + + public static List FindMultiTarget(Mat srcMat, Mat dstMat, string title, out Mat resMat, + double threshold = 0.8, int findTargetCount = 8) + { + List pointList = new List(); + resMat = srcMat.Clone(); + try + { + dstMat = ResizeHelper.Resize(dstMat, WidthScale); + + Mat matchResult = new Mat(); + Cv2.MatchTemplate(srcMat, dstMat, matchResult, TemplateMatchModes.CCoeffNormed); + + double minValue = 0; + double maxValue = 0; + Point minLoc = new(); + + //寻找最几个最值的位置 + Mat mask = new Mat(matchResult.Height, matchResult.Width, MatType.CV_8UC1, Scalar.White); + Mat maskSub = new Mat(matchResult.Height, matchResult.Width, MatType.CV_8UC1, Scalar.Black); + var point = new OpenCvSharp.Point(0, 0); + for (int i = 0; i < findTargetCount; i++) + { + Cv2.MinMaxLoc(matchResult, out minValue, out maxValue, out minLoc, out point, mask); + Rect maskRect = new Rect(point.X - dstMat.Width / 2, point.Y - dstMat.Height / 2, dstMat.Width, + dstMat.Height); + maskSub.Rectangle(maskRect, Scalar.White, -1); + mask -= maskSub; + if (maxValue >= threshold) + { + pointList.Add(new Point(point.X + dstMat.Width / 2, point.Y + dstMat.Height / 2)); + + //if (VisionContext.Instance().Drawable) + //{ + // VisionContext.Instance().DrawContent.RectList + // .Add(new System.Windows.Rect(point.X, point.Y, dstMat.Width, dstMat.Height)); + // VisionContext.Instance().DrawContent.TextList + // .Add(new Tuple(new System.Windows.Point(point.X, point.Y - 10), maxValue.ToString("0.00"))); + //} + //if (IsDebug) + //{ + // VisionContext.Instance().Log + // ?.LogInformation(title + " " + maxValue.ToString("0.000") + " " + point); + // Cv2.Rectangle(resMat, point, + // new OpenCvSharp.Point(point.X + dstMat.Width, point.Y + dstMat.Height), + // Scalar.Red, 2); + // Cv2.PutText(resMat, title + " " + maxValue.ToString("0.00"), + // new OpenCvSharp.Point(point.X, point.Y - 10), + // HersheyFonts.HersheySimplex, 0.5, Scalar.Red); + //} + } + else + { + break; + } + } + + return pointList; + } + catch (Exception ex) + { + _logger.LogError(ex.ToString()); + return pointList; + } + finally + { + srcMat?.Dispose(); + dstMat?.Dispose(); + } + } + + + public static Dictionary> FindMultiPicFromOneImage(Bitmap imgSrc, + Dictionary imgSubDictionary, double threshold = 0.8) + { + Dictionary> dictionary = new Dictionary>(); + Mat srcMat = imgSrc.ToMat(); + Mat resMat; + + foreach (KeyValuePair kvp in imgSubDictionary) + { + dictionary.Add(kvp.Key, FindMultiTarget(srcMat, kvp.Value.ToMat(), kvp.Key, out resMat, threshold)); + srcMat = resMat.Clone(); + } + + return dictionary; + } + + public static Dictionary> FindMultiPicFromOneImage(Mat srcMat, + Dictionary imgSubDictionary, double threshold = 0.8) + { + Dictionary> dictionary = new Dictionary>(); + Mat resMat; + foreach (KeyValuePair kvp in imgSubDictionary) + { + dictionary.Add(kvp.Key, FindMultiTarget(srcMat, kvp.Value.ToMat(), kvp.Key, out resMat, threshold)); + srcMat = resMat.Clone(); + } + + return dictionary; + } + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/OpenCv/OpenCvCommonHelper.cs b/BetterGenshinImpact/Core/Recognition/OpenCv/OpenCvCommonHelper.cs index 9335e1e9..7ba1e040 100644 --- a/BetterGenshinImpact/Core/Recognition/OpenCv/OpenCvCommonHelper.cs +++ b/BetterGenshinImpact/Core/Recognition/OpenCv/OpenCvCommonHelper.cs @@ -47,5 +47,21 @@ namespace BetterGenshinImpact.Core.Recognition.OpenCv } return sum; } + + public static Mat Threshold(Mat src, Scalar low, Scalar high) + { + using var mask = new Mat(); + using var rgbMat = new Mat(); + + Cv2.CvtColor(src, rgbMat, ColorConversionCodes.BGR2RGB); + Cv2.InRange(rgbMat, low, high, mask); + Cv2.Threshold(mask, mask, 0, 255, ThresholdTypes.Binary); //二值化 + return mask; + } + + public static Mat Threshold(Mat src, Scalar s) + { + return Threshold(src, s, s); + } } } \ No newline at end of file diff --git a/BetterGenshinImpact/Core/Recognition/RecognitionObject.cs b/BetterGenshinImpact/Core/Recognition/RecognitionObject.cs index d9b4cf8c..74ede9ee 100644 --- a/BetterGenshinImpact/Core/Recognition/RecognitionObject.cs +++ b/BetterGenshinImpact/Core/Recognition/RecognitionObject.cs @@ -2,16 +2,19 @@ using System.Collections.Generic; using System.Drawing; using System.Linq; +using System.Security.Cryptography.X509Certificates; using System.Text; using System.Threading.Tasks; +using BetterGenshinImpact.Core.Recognition.OpenCv; +using BetterGenshinImpact.Helpers.Extensions; using OpenCvSharp; -using Rect = System.Windows.Rect; namespace BetterGenshinImpact.Core.Recognition; /// /// 识别对象 /// +[Serializable] public class RecognitionObject { public RecognitionType RecognitionType { get; set; } @@ -23,9 +26,15 @@ public class RecognitionObject #region 模板匹配 /// - /// 模板匹配的对象 + /// 模板匹配的对象(彩色) /// - public Mat TemplateImageMat { get; set; } + public Mat? TemplateImageMat { get; set; } + + /// + /// 模板匹配的对象(灰色) + /// + public Mat? TemplateImageGreyMat { get; set; } + /// /// 模板匹配阈值。可选，默认 0.8 。 /// @@ -44,10 +53,37 @@ public class RecognitionObject public bool UseMask { get; set; } = false; /// - /// 不需要匹配的颜色，默认绿幕 + /// 不需要匹配的颜色，默认绿色 + /// UseMask = true 的时候有用 /// public Color MaskColor { get; set; } = Color.FromArgb(0, 255, 0); + public Mat? MaskMat { get; set; } + + /// + /// 匹配成功时，是否在屏幕上绘制矩形框。可选，默认 false 。 + /// true 时 Name 必须有值。 + /// + public bool DrawOnWindow { get; set; } = false; + + /// + /// DrawOnWindow 为 true 时，绘制的矩形框的颜色。可选，默认红色。 + /// + public Pen DrawOnWindowPen = new(Color.Red, 2); + + public void InitTemplate() + { + if (TemplateImageMat != null && TemplateImageGreyMat == null) + { + TemplateImageGreyMat = new Mat(); + Cv2.CvtColor(TemplateImageMat, TemplateImageGreyMat, ColorConversionCodes.BGR2GRAY); + } + + if (UseMask && TemplateImageMat != null && MaskMat == null) + { + MaskMat = OpenCvCommonHelper.Threshold(TemplateImageMat, MaskColor.ToScalar()); + } + } #endregion @@ -62,6 +98,7 @@ public class RecognitionObject public Color LowerColor { get; set; } public Color UpperColor { get; set; } + /// /// 符合的点的数量要求。可选，默认 1 /// @@ -104,6 +141,7 @@ public class RecognitionObject /// /// 圣遗物 Yas /// 拾取 Yap + /// TODO 换成枚举 /// public string? ModelTextRecognitionType { get; set; } diff --git a/BetterGenshinImpact/Core/Simulator/MouseEventSimulator.cs b/BetterGenshinImpact/Core/Simulator/MouseEventSimulator.cs index da9e25da..5cac39b1 100644 --- a/BetterGenshinImpact/Core/Simulator/MouseEventSimulator.cs +++ b/BetterGenshinImpact/Core/Simulator/MouseEventSimulator.cs @@ -1,6 +1,7 @@ using System.Threading; using System.Windows; using Windows.Win32.UI.Input.KeyboardAndMouse; +using BetterGenshinImpact.Helpers; using static Windows.Win32.PInvoke; namespace BetterGenshinImpact.Core.Simulator; diff --git a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs index e650160d..25dedbe3 100644 --- a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs +++ b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs @@ -143,7 +143,7 @@ namespace BetterGenshinImpact.GameTask.AutoFishing { var grayMat = content.SrcGreyMat; var grayRightBottomMat = CutHelper.CutRightBottom(grayMat, grayMat.Width / 3, grayMat.Height / 5); - var p = MatchTemplateHelper.FindSingleTarget(grayRightBottomMat, AutoFishingAssets.SpaceButtonMat); + var p = OldMatchTemplateHelper.FindSingleTarget(grayRightBottomMat, AutoFishingAssets.SpaceButtonMat); return p is { X: > 0, Y: > 0 }; } diff --git a/BetterGenshinImpact/GameTask/AutoPick/AutoPickTrigger.cs b/BetterGenshinImpact/GameTask/AutoPick/AutoPickTrigger.cs index 31afa7ed..f0b3a4c9 100644 --- a/BetterGenshinImpact/GameTask/AutoPick/AutoPickTrigger.cs +++ b/BetterGenshinImpact/GameTask/AutoPick/AutoPickTrigger.cs @@ -21,7 +21,7 @@ namespace BetterGenshinImpact.GameTask.AutoPick public void OnCapture(CaptureContent content) { var grayRightBottomMat = content.SrcGreyRightBottomMat.Clone(); - var p2 = MatchTemplateHelper.FindSingleTarget(grayRightBottomMat, AutoPickAssets.FMat); + var p2 = OldMatchTemplateHelper.FindSingleTarget(grayRightBottomMat, AutoPickAssets.FMat); if (p2 is { X: > 0, Y: > 0 }) { _logger.LogInformation("找到F按钮"); diff --git a/BetterGenshinImpact/GameTask/AutoSkip/Assets/AutoSkipAssets.cs b/BetterGenshinImpact/GameTask/AutoSkip/Assets/AutoSkipAssets.cs index 5370213a..847776ba 100644 --- a/BetterGenshinImpact/GameTask/AutoSkip/Assets/AutoSkipAssets.cs +++ b/BetterGenshinImpact/GameTask/AutoSkip/Assets/AutoSkipAssets.cs @@ -1,17 +1,43 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BetterGenshinImpact.Core.Config; +using BetterGenshinImpact.Core.Recognition; using OpenCvSharp; -namespace BetterGenshinImpact.GameTask.AutoSkip.Assets +namespace BetterGenshinImpact.GameTask.AutoSkip.Assets; + +public class AutoSkipAssets { - public class AutoSkipAssets + public RecognitionObject StopAutoButtonRo; + public RecognitionObject OptionButtonRo; + public RecognitionObject MenuRo; + + public AutoSkipAssets() { - public static Mat StopAutoButtonMat = new(Global.Absolute(@"GameTask\AutoSkip\Assets\1920x1080\stop_auto.png"), ImreadModes.Grayscale); - public static Mat OptionMat = new(Global.Absolute(@"GameTask\AutoSkip\Assets\1920x1080\option.png"), ImreadModes.Grayscale); - public static Mat MenuMat = new(Global.Absolute(@"GameTask\AutoSkip\Assets\1920x1080\menu.png"), ImreadModes.Grayscale); + var info = TaskContext.Instance().SystemInfo; + StopAutoButtonRo = new RecognitionObject + { + Name = "StopAutoButton", + RecognitionType = RecognitionType.TemplateMatch, + TemplateImageMat = GameTaskManager.LoadAssertImage("AutoSkip", "stop_auto.png"), + RegionOfInterest = new Rect(0, 0, info.GameScreenSize.Width / 5, info.GameScreenSize.Height / 5), + DrawOnWindow = true + }; + StopAutoButtonRo.InitTemplate(); + OptionButtonRo = new RecognitionObject + { + Name = "OptionButton", + RecognitionType = RecognitionType.TemplateMatch, + TemplateImageMat = GameTaskManager.LoadAssertImage("AutoSkip", "option.png"), + RegionOfInterest = new Rect(info.GameScreenSize.Width / 2, 0, info.GameScreenSize.Width - info.GameScreenSize.Width / 2, info.GameScreenSize.Height), + DrawOnWindow = true + }; + OptionButtonRo.InitTemplate(); + MenuRo = new RecognitionObject + { + Name = "Menu", + RecognitionType = RecognitionType.TemplateMatch, + TemplateImageMat = GameTaskManager.LoadAssertImage("AutoSkip", "menu.png"), + RegionOfInterest = new Rect(0, 0, info.GameScreenSize.Width / 4, info.GameScreenSize.Height / 4), + DrawOnWindow = true + }; + MenuRo.InitTemplate(); } -} +} \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs b/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs index 22449b55..62194771 100644 --- a/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs +++ b/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs @@ -5,13 +5,14 @@ using BetterGenshinImpact.View.Drawable; using Microsoft.Extensions.Logging; using System; using System.Diagnostics; +using BetterGenshinImpact.GameTask.Model; using WindowsInput; using Point = OpenCvSharp.Point; namespace BetterGenshinImpact.GameTask.AutoSkip { /// - /// 自动剧情有选项点击，必须使用BitBlt + /// 自动剧情有选项点击 /// public class AutoSkipTrigger : ITaskTrigger { @@ -22,6 +23,13 @@ namespace BetterGenshinImpact.GameTask.AutoSkip public int Priority => 20; public bool IsExclusive => false; + private AutoSkipAssets autoSkipAssets; + + public AutoSkipTrigger() + { + autoSkipAssets = new AutoSkipAssets(); + } + public void Init() { IsEnabled = true; @@ -34,50 +42,55 @@ namespace BetterGenshinImpact.GameTask.AutoSkip return; } - var grayMat = content.SrcGreyMat; - // 找左上角剧情自动的按钮 - var grayLeftTopMat = CutHelper.CutLeftTop(grayMat, grayMat.Width / 5, grayMat.Height / 5); - var p1 = MatchTemplateHelper.FindSingleTarget(grayLeftTopMat, AutoSkipAssets.StopAutoButtonMat, 0.9); - if (p1 is { X: > 0, Y: > 0 }) + new RectArea().Find(autoSkipAssets.StopAutoButtonRo, (_) => { - //_logger.LogInformation($"找到剧情自动按钮：{p1}"); - VisionContext.Instance().DrawContent.PutRect("StopAutoButton", - p1.CenterPointToRect(AutoSkipAssets.StopAutoButtonMat).ToRectDrawable()); new InputSimulator().Keyboard.KeyPress(VirtualKeyCode.SPACE); - } - else - { - VisionContext.Instance().DrawContent.RemoveRect("StopAutoButton"); - } + }); - // 不存在则找右下的选项按钮 - var grayRightBottomMat = content.SrcGreyRightBottomMat; - var p2 = MatchTemplateHelper.FindSingleTarget(grayRightBottomMat, AutoSkipAssets.OptionMat); - if (p2 is { X: > 0, Y: > 0 }) - { - // 不存在菜单的情况下 剧情在播放中 - var grayLeftTopMat2 = CutHelper.CutLeftTop(grayMat, grayMat.Width / 4, grayMat.Height / 4); - var pMenu = MatchTemplateHelper.FindSingleTarget(grayLeftTopMat2, AutoSkipAssets.MenuMat); - if (pMenu is { X: 0, Y: 0 }) - { - p2 = p2.ToDesktopPositionOffset65535(grayMat.Width - grayMat.Width / 2, - grayMat.Height - grayMat.Height / 3 * 2); - new InputSimulator().Mouse.MoveMouseTo(p2.X, p2.Y).LeftButtonClick(); - _logger.LogInformation($"点击选项按钮：{p2}"); - return; - } - } + //var grayMat = content.SrcGreyMat; + //// 找左上角剧情自动的按钮 + //var grayLeftTopMat = CutHelper.CutLeftTop(grayMat, grayMat.Width / 5, grayMat.Height / 5); + //var p1 = OldMatchTemplateHelper.FindSingleTarget(grayLeftTopMat, AutoSkipAssets.StopAutoButtonMat, 0.9); + //if (p1 is { X: > 0, Y: > 0 }) + //{ + // //_logger.LogInformation($"找到剧情自动按钮：{p1}"); + // VisionContext.Instance().DrawContent.PutRect("StopAutoButton", + // p1.CenterPointToRect(AutoSkipAssets.StopAutoButtonMat).ToRectDrawable()); + // new InputSimulator().Keyboard.KeyPress(VirtualKeyCode.SPACE); + //} + //else + //{ + // VisionContext.Instance().DrawContent.RemoveRect("StopAutoButton"); + //} - // 黑屏剧情要点击鼠标（多次） 几乎全黑的时候不用点击 - var blackCount = OpenCvCommonHelper.CountGrayMatColor(grayMat, 0); - var rate = blackCount * 1.0 / (grayMat.Width * grayMat.Height); - if (rate > 0.7 && rate < 0.99) - { - var p3 = new Point(grayMat.Width / 2, grayMat.Height / 2).ToDesktopPosition65535(); - new InputSimulator().Mouse.MoveMouseTo(p3.X, p3.Y).LeftButtonClick(); - Debug.WriteLine($"点击黑屏剧情：{rate}"); - return; - } + //// 不存在则找右下的选项按钮 + //var grayRightBottomMat = content.SrcGreyRightBottomMat; + //var p2 = OldMatchTemplateHelper.FindSingleTarget(grayRightBottomMat, AutoSkipAssets.OptionMat); + //if (p2 is { X: > 0, Y: > 0 }) + //{ + // // 不存在菜单的情况下 剧情在播放中 + // var grayLeftTopMat2 = CutHelper.CutLeftTop(grayMat, grayMat.Width / 4, grayMat.Height / 4); + // var pMenu = OldMatchTemplateHelper.FindSingleTarget(grayLeftTopMat2, AutoSkipAssets.MenuMat); + // if (pMenu is { X: 0, Y: 0 }) + // { + // p2 = p2.ToDesktopPositionOffset65535(grayMat.Width - grayMat.Width / 2, + // grayMat.Height - grayMat.Height / 3 * 2); + // new InputSimulator().Mouse.MoveMouseTo(p2.X, p2.Y).LeftButtonClick(); + // _logger.LogInformation($"点击选项按钮：{p2}"); + // return; + // } + //} + + //// 黑屏剧情要点击鼠标（多次） 几乎全黑的时候不用点击 + //var blackCount = OpenCvCommonHelper.CountGrayMatColor(grayMat, 0); + //var rate = blackCount * 1.0 / (grayMat.Width * grayMat.Height); + //if (rate > 0.7 && rate < 0.99) + //{ + // var p3 = new Point(grayMat.Width / 2, grayMat.Height / 2).ToDesktopPosition65535(); + // new InputSimulator().Mouse.MoveMouseTo(p3.X, p3.Y).LeftButtonClick(); + // Debug.WriteLine($"点击黑屏剧情：{rate}"); + // return; + //} // TODO 自动交付材料 } } diff --git a/BetterGenshinImpact/GameTask/CaptureContent.cs b/BetterGenshinImpact/GameTask/CaptureContent.cs index 26ac5681..21eed20c 100644 --- a/BetterGenshinImpact/GameTask/CaptureContent.cs +++ b/BetterGenshinImpact/GameTask/CaptureContent.cs @@ -1,6 +1,7 @@ using System; using System.Drawing; using BetterGenshinImpact.Core.Recognition.OpenCv; +using BetterGenshinImpact.GameTask.Model; using OpenCvSharp; namespace BetterGenshinImpact.GameTask; @@ -17,11 +18,14 @@ public class CaptureContent public int FrameIndex { get; private set; } public int FrameRate { get; private set; } - public CaptureContent(Bitmap srcBitmap, int frameIndex, int frameRate) + public RectArea CaptureRectArea { get; private set; } + + public CaptureContent(Bitmap srcBitmap, int frameIndex, int frameRate, RectArea rectArea) { SrcBitmap = srcBitmap; FrameIndex = frameIndex; FrameRate = frameRate; + CaptureRectArea = rectArea; } private Mat? _srcMat; diff --git a/BetterGenshinImpact/GameTask/GameTaskManager.cs b/BetterGenshinImpact/GameTask/GameTaskManager.cs index f6210a85..d81bc162 100644 --- a/BetterGenshinImpact/GameTask/GameTaskManager.cs +++ b/BetterGenshinImpact/GameTask/GameTaskManager.cs @@ -1,5 +1,9 @@ -using System.Collections.Generic; +using BetterGenshinImpact.Core.Config; +using OpenCvSharp; +using System.Collections.Generic; +using System.IO; using System.Linq; +using BetterGenshinImpact.Core.Recognition.OpenCv; namespace BetterGenshinImpact.GameTask { @@ -9,9 +13,9 @@ namespace BetterGenshinImpact.GameTask { List loadedTriggers = new() { - new AutoPick.AutoPickTrigger(), + //new AutoPick.AutoPickTrigger(), new AutoSkip.AutoSkipTrigger(), - new AutoFishing.AutoFishingTrigger() + //new AutoFishing.AutoFishingTrigger() }; loadedTriggers.ForEach(i => i.Init()); @@ -19,5 +23,37 @@ namespace BetterGenshinImpact.GameTask return loadedTriggers.OrderByDescending(i => i.Priority).ToList(); } + /// + /// 获取素材图片并缩放 + /// todo 支持多语言 + /// + /// 任务名称 + /// 素材文件名 + /// + /// + public static Mat LoadAssertImage(string featName, string assertName) + { + var info = TaskContext.Instance().SystemInfo; + var assetsFolder = Global.Absolute($@"GameTask\AutoSkip\Assets\{info.GameScreenSize.Width}x{info.GameScreenSize.Height}"); + if (!Directory.Exists(assetsFolder)) + { + assetsFolder = Global.Absolute($@"GameTask\AutoSkip\Assets\1920x1080"); + } + if (!Directory.Exists(assetsFolder)) + { + throw new FileNotFoundException($"未找到{featName}的素材文件夹"); + } + var filePath = Path.Combine(assetsFolder, assertName); + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"未找到{featName}中的{assertName}文件"); + } + var mat = new Mat(filePath, ImreadModes.AnyColor); + if (info.GameScreenSize.Width != 1920) + { + mat = ResizeHelper.Resize(mat, info.AssetScale); + } + return mat; + } } -} +} \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/Model/RectArea.cs b/BetterGenshinImpact/GameTask/Model/RectArea.cs index 68cbb658..47a967fb 100644 --- a/BetterGenshinImpact/GameTask/Model/RectArea.cs +++ b/BetterGenshinImpact/GameTask/Model/RectArea.cs @@ -1,16 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Drawing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Xml.Linq; +using BetterGenshinImpact.Core.Recognition; using BetterGenshinImpact.Core.Recognition.OpenCv; -using BetterGenshinImpact.GameTask.AutoSkip.Assets; using BetterGenshinImpact.Helpers.Extensions; +using BetterGenshinImpact.View.Drawable; using OpenCvSharp; -using WindowsInput; -using static Vanara.PInvoke.Gdi32; +using System; +using System.Drawing; using Point = OpenCvSharp.Point; namespace BetterGenshinImpact.GameTask.Model; @@ -165,6 +159,11 @@ public class RectArea return ConvertRelativePositionTo(0); } + public Rect ConvertRelativePositionToCaptureArea() + { + return ConvertRelativePositionTo(1); + } + public Rect ToRect() { return new Rect(X, Y, Width, Height); @@ -190,21 +189,101 @@ public class RectArea /// /// /// + [Obsolete] public RectArea Find(Mat targetImageMat) { if (!HasImage()) { throw new Exception("当前对象内没有图像内容，无法完成 Find 操作"); } - var p = MatchTemplateHelper.FindSingleTarget(SrcGreyMat, targetImageMat); + + var p = OldMatchTemplateHelper.FindSingleTarget(SrcGreyMat, targetImageMat); return p is { X: > 0, Y: > 0 } ? new RectArea(targetImageMat, p.X - targetImageMat.Width / 2, p.Y - targetImageMat.Height / 2, this) : new RectArea(); } + /// + /// 在本区域内查找识别对象 + /// + /// + /// + /// + /// + public RectArea Find(RecognitionObject ro, Action? action = null) + { + if (!HasImage()) + { + throw new Exception("当前对象内没有图像内容，无法完成 Find 操作"); + } + + if (ro == null) + { + throw new Exception("识别对象不能为null"); + } + + if (RecognitionType.TemplateMatch.Equals(ro.RecognitionType)) + { + if (ro.TemplateImageGreyMat == null) + { + throw new Exception("识别对象的模板图片不能为null"); + } + + var roi = SrcGreyMat; + if (ro.RegionOfInterest != Rect.Empty) + { + roi = new Mat(SrcGreyMat, ro.RegionOfInterest); + } + + var p = MatchTemplateHelper.MatchTemplate(roi, ro.TemplateImageGreyMat, ro.TemplateMatchMode, ro.MaskMat, ro.Threshold); + if (p is { X: > 0, Y: > 0 }) + { + var newRa = new RectArea(ro.TemplateImageGreyMat, p.X + ro.RegionOfInterest.X, p.Y + ro.RegionOfInterest.Y, this); + if (ro.DrawOnWindow && !string.IsNullOrEmpty(ro.Name)) + { + VisionContext.Instance().DrawContent.PutRect(ro.Name, newRa + .ConvertRelativePositionToCaptureArea() + .ToRectDrawable(ro.DrawOnWindowPen, ro.Name)); + } + action?.Invoke(newRa); + return newRa; + } + else + { + if (ro.DrawOnWindow && !string.IsNullOrEmpty(ro.Name)) + { + VisionContext.Instance().DrawContent.RemoveRect(ro.Name); + } + + return new RectArea(); + } + } + else + { + throw new Exception($"RectArea不支持的识别类型{ro.RecognitionType}"); + } + } + + /// + /// 找到识别对象并点击中心 + /// + /// + /// + public RectArea ClickCenter(RecognitionObject ro) + { + var ra = Find(ro); + if (!ra.IsEmpty()) + { + ra.ClickCenter(); + } + + return ra; + } + /// /// 找到图像并点击中心 /// /// /// + [Obsolete] public RectArea ClickCenter(Mat targetImageMat) { var ra = Find(targetImageMat); diff --git a/BetterGenshinImpact/GameTask/Model/SystemInfo.cs b/BetterGenshinImpact/GameTask/Model/SystemInfo.cs new file mode 100644 index 00000000..62095346 --- /dev/null +++ b/BetterGenshinImpact/GameTask/Model/SystemInfo.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Windows.Win32.Foundation; + +namespace BetterGenshinImpact.GameTask.Model +{ + public class SystemInfo + { + /// + /// 显示器分辨率 无缩放 + /// + public Size DisplaySize { get; set; } + + public string GameProcessName { get; set; } + + public int GameProcessId { get; set; } + + public nint GameProcessHandle { get; set; } + + /// + /// 游戏窗口内分辨率 + /// + public RECT GameScreenSize { get; set; } + + /// + /// 素材缩放比例 + /// + public double AssetScale { get; set; } = 1; + } +} diff --git a/BetterGenshinImpact/GameTask/SystemControl.cs b/BetterGenshinImpact/GameTask/SystemControl.cs index e247b183..f4e14097 100644 --- a/BetterGenshinImpact/GameTask/SystemControl.cs +++ b/BetterGenshinImpact/GameTask/SystemControl.cs @@ -68,7 +68,7 @@ namespace BetterGenshinImpact.GameTask /// public static RECT GetGameScreenRect(HWND hWnd) { - Windows.Win32.PInvoke.GetWindowRect(hWnd, out var clientRect); + GetClientRect(hWnd, out var clientRect); return clientRect; } @@ -76,5 +76,7 @@ namespace BetterGenshinImpact.GameTask //{ // return User32.GetSystemMetrics(User32.SystemMetric.SM_CYFRAME) + User32.GetSystemMetrics(User32.SystemMetric.SM_CYCAPTION); //} + + } } \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/TaskContext.cs b/BetterGenshinImpact/GameTask/TaskContext.cs index 8173c9ef..4a59279e 100644 --- a/BetterGenshinImpact/GameTask/TaskContext.cs +++ b/BetterGenshinImpact/GameTask/TaskContext.cs @@ -1,4 +1,5 @@ -using System; +using BetterGenshinImpact.GameTask.Model; +using System; namespace BetterGenshinImpact.GameTask { @@ -28,5 +29,7 @@ namespace BetterGenshinImpact.GameTask } public IntPtr GameHandle { get; set; } + + public SystemInfo SystemInfo { get; set; } } } diff --git a/BetterGenshinImpact/GameTask/TaskDispatcher.cs b/BetterGenshinImpact/GameTask/TaskDispatcher.cs index c0ce384a..2669b34d 100644 --- a/BetterGenshinImpact/GameTask/TaskDispatcher.cs +++ b/BetterGenshinImpact/GameTask/TaskDispatcher.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading; using System.Windows; using Windows.Win32.Foundation; +using BetterGenshinImpact.GameTask.Model; using Vision.WindowCapture; namespace BetterGenshinImpact.GameTask @@ -19,7 +20,7 @@ namespace BetterGenshinImpact.GameTask private IWindowCapture? _capture; - private static object _locker = new(); + private static readonly object _locker = new(); private int _frameIndex = 0; private int _frameRate = 30; @@ -102,7 +103,7 @@ namespace BetterGenshinImpact.GameTask } // 循环执行所有触发器 有独占状态的触发器的时候只执行独占触发器 - var content = new CaptureContent(bitmap, _frameIndex, _frameRate); + var content = new CaptureContent(bitmap, _frameIndex, _frameRate, new RectArea()); var exclusiveTrigger = _triggers.FirstOrDefault(t => t is { IsEnabled: true, IsExclusive: true }); if (exclusiveTrigger != null) { diff --git a/BetterGenshinImpact/Helpers/Extensions/BitmapExtension.cs b/BetterGenshinImpact/Helpers/Extensions/BitmapExtension.cs index 09e1034e..59a192f7 100644 --- a/BetterGenshinImpact/Helpers/Extensions/BitmapExtension.cs +++ b/BetterGenshinImpact/Helpers/Extensions/BitmapExtension.cs @@ -1,11 +1,7 @@ -using System; -using System.Collections.Generic; -using System.Drawing; +using System.Drawing; using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows.Media.Imaging; +using OpenCvSharp; namespace BetterGenshinImpact.Helpers.Extensions { @@ -23,5 +19,10 @@ namespace BetterGenshinImpact.Helpers.Extensions image.EndInit(); return image; } + + public static Scalar ToScalar(this Color color) + { + return new Scalar(color.R, color.G, color.B); + } } } diff --git a/BetterGenshinImpact/Core/Simulator/PrimaryScreen.cs b/BetterGenshinImpact/Helpers/PrimaryScreen.cs similarity index 94% rename from BetterGenshinImpact/Core/Simulator/PrimaryScreen.cs rename to BetterGenshinImpact/Helpers/PrimaryScreen.cs index e77b559a..f9644bec 100644 --- a/BetterGenshinImpact/Core/Simulator/PrimaryScreen.cs +++ b/BetterGenshinImpact/Helpers/PrimaryScreen.cs @@ -1,11 +1,8 @@ -using System; -using System.Drawing; -using System.Runtime.InteropServices; -using Windows.Win32.Foundation; +using System.Drawing; using Windows.Win32.Graphics.Gdi; using static Windows.Win32.PInvoke; -namespace BetterGenshinImpact.Core.Simulator +namespace BetterGenshinImpact.Helpers { public class PrimaryScreen {