From 9b5bfd6dfbc669c976d774a7927ffd2fcf33546c Mon Sep 17 00:00:00 2001 From: huiyadanli Date: Sun, 24 Sep 2023 19:18:25 +0800 Subject: [PATCH] feat: finsh auto fish --- .../BetterGenshinImpact.csproj | 1 - .../AutoFishingImageRecognition.cs | 39 +++ .../AutoFishing/AutoFishingTrigger.cs | 227 ++++++++++++++++++ .../GameTask/AutoSkip/AutoSkipTrigger.cs | 3 + .../GameTask/GameTaskManager.cs | 7 +- .../GameTask/TaskDispatcher.cs | 4 +- BetterGenshinImpact/View/MainWindow.xaml | 2 +- Vision.Recognition/DrawContent.cs | 39 ++- .../Helper/OpenCv/CommonExtension.cs | 5 + Vision.Recognition/MaskWindow.xaml.cs | 19 +- Vision.Recognition/Task/CaptureContent.cs | 4 +- 11 files changed, 336 insertions(+), 14 deletions(-) create mode 100644 BetterGenshinImpact/GameTask/AutoFishing/AutoFishingImageRecognition.cs create mode 100644 BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs diff --git a/BetterGenshinImpact/BetterGenshinImpact.csproj b/BetterGenshinImpact/BetterGenshinImpact.csproj index 2af69aaa..76a33305 100644 --- a/BetterGenshinImpact/BetterGenshinImpact.csproj +++ b/BetterGenshinImpact/BetterGenshinImpact.csproj @@ -25,7 +25,6 @@ - diff --git a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingImageRecognition.cs b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingImageRecognition.cs new file mode 100644 index 00000000..e37c318f --- /dev/null +++ b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingImageRecognition.cs @@ -0,0 +1,39 @@ +using OpenCvSharp; +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Vision.Recognition.Helper.OpenCv; + +namespace BetterGenshinImpact.GameTask.AutoFishing +{ + public class AutoFishingImageRecognition + { + /// + /// 钓鱼条矩形识别 + /// + /// + /// + public static List? GetFishBarRect(Mat src) + { + using var mask = new Mat(); + using var rgbMat = new Mat(); + + Cv2.CvtColor(src, rgbMat, ColorConversionCodes.BGR2RGB); + var lowPurple = new Scalar(255, 255, 192); + var highPurple = new Scalar(255, 255, 192); + Cv2.InRange(rgbMat, lowPurple, highPurple, mask); + Cv2.Threshold(mask, mask, 0, 255, ThresholdTypes.Binary); //二值化 + + Cv2.FindContours(mask, out var contours, out _, RetrievalModes.External, ContourApproximationModes.ApproxSimple, null); + if (contours.Length > 0) + { + var boxes = contours.Select(Cv2.BoundingRect).Where(w => w.Height >= 10); + return boxes.ToList(); + } + return null; + } + } +} diff --git a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs new file mode 100644 index 00000000..7d319d84 --- /dev/null +++ b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTrigger.cs @@ -0,0 +1,227 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.ConstrainedExecution; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Media.Media3D; +using OpenCvSharp; +using Vision.Recognition; +using Vision.Recognition.Helper.OpenCv; +using Vision.Recognition.Task; +using WindowsInput; +using static Vanara.PInvoke.Kernel32; +using static Vanara.PInvoke.User32; +using WinRT; +using Windows.ApplicationModel.Contacts; + +namespace BetterGenshinImpact.GameTask.AutoFishing +{ + public class AutoFishingTrigger : ITaskTrigger + { + private readonly ILogger _logger = App.GetLogger(); + + public string Name => "自动钓鱼"; + public bool IsEnabled { get; set; } + public int Priority => 15; + + /// + /// 钓鱼是要独占模式的 + /// 在钓鱼的时候，不应该有其他任务在执行 + /// 在触发器发现正在钓鱼的时候，启用独占模式 + /// + public bool IsExclusive { get; set; } + + public void Init() + { + IsEnabled = true; + IsExclusive = true; + + // 钓鱼变量初始化 + _findFishBoxTips = false; + } + + private Rect _fishBoxRect = new(0, 0, 0, 0); + + public void OnCapture(CaptureContent content) + { + // TODO 进入独占的判定 + + if (_fishBoxRect.Width == 0) + { + GetFishBoxArea(content.SrcMat); + } + else + { + Fishing(content, new Mat(content.SrcMat, _fishBoxRect)); + } + } + + + /// + /// 获取钓鱼框的位置 + /// + private void GetFishBoxArea(Mat srcMat) + { + var rects = AutoFishingImageRecognition.GetFishBarRect(srcMat); + if (rects != null && rects.Count == 2) + { + if (Math.Abs(rects[0].Height - rects[1].Height) > 10) + { + Debug.WriteLine("两个矩形高度差距过大，未识别到钓鱼框"); + return; + } + + if (rects[0].Width < rects[1].Width) + { + _cur = rects[0]; + _left = rects[1]; + } + else + { + _cur = rects[1]; + _left = rects[0]; + } + + // cur 是游标位置, 在初始状态下，cur 一定在left左边 + if (_left.X < _cur.X) + { + return; + } + + int hExtra = _cur.Height, vExtra = _cur.Height / 4; + _fishBoxRect = new Rect(_cur.X - hExtra, _cur.Y - vExtra, + (_left.X + _left.Width / 2 - _cur.X) * 2 + hExtra * 2, _cur.Height + vExtra * 2); + VisionContext.Instance().DrawContent.PutRect("FishBox", _fishBoxRect.ToWindowsRectangle()); + } + } + + + private int _noRectsCount = 0; + private bool _isFishingProcess = false; // 提杆后会设置为true + private Rect _cur, _left, _right; + private MOUSEEVENTF _prevMouseEvent = 0x0; + private bool _findFishBoxTips; + + + /// + /// 钓鱼拉条 + /// + /// + /// + private void Fishing(CaptureContent content, Mat fishBarMat) + { + List? rects = AutoFishingImageRecognition.GetFishBarRect(fishBarMat); + if (rects != null && rects.Count > 0) + { + var simulator = new InputSimulator(); + if (rects.Count >= 2 && _prevMouseEvent == 0x0 && !_findFishBoxTips) + { + _findFishBoxTips = true; + _logger.LogInformation(" 识别到钓鱼框，自动拉扯中..."); + } + + // 超过3个矩形是异常情况，取高度最高的三个矩形进行识别 + if (rects.Count > 3) + { + rects.Sort((a, b) => b.Height.CompareTo(a.Height)); + rects.RemoveRange(3, rects.Count - 3); + } + + Debug.WriteLine($"识别到{rects.Count} 个矩形"); + if (rects.Count == 2) + { + if (rects[0].Width < rects[1].Width) + { + _cur = rects[0]; + _left = rects[1]; + } + else + { + _cur = rects[1]; + _left = rects[0]; + } + + PutRects(_left, _cur, new Rect()); + + if (_cur.X < _left.X) + { + if (_prevMouseEvent != MOUSEEVENTF.MOUSEEVENTF_LEFTDOWN) + { + simulator.Mouse.LeftButtonDown(); + _prevMouseEvent = MOUSEEVENTF.MOUSEEVENTF_LEFTDOWN; + Debug.WriteLine("进度不到 左键按下"); + } + } + else + { + if (_prevMouseEvent == MOUSEEVENTF.MOUSEEVENTF_LEFTDOWN) + { + simulator.Mouse.LeftButtonUp(); + _prevMouseEvent = MOUSEEVENTF.MOUSEEVENTF_LEFTUP; + Debug.WriteLine("进度超出 左键松开"); + } + } + } + else if (rects.Count == 3) + { + rects.Sort((a, b) => a.X.CompareTo(b.X)); + _left = rects[0]; + _cur = rects[1]; + _right = rects[2]; + PutRects(_left, _cur, _right); + + if (_right.X + _right.Width - (_cur.X + _cur.Width) <= _cur.X - _left.X) + { + if (_prevMouseEvent == MOUSEEVENTF.MOUSEEVENTF_LEFTDOWN) + { + simulator.Mouse.LeftButtonUp(); + _prevMouseEvent = MOUSEEVENTF.MOUSEEVENTF_LEFTUP; + Debug.WriteLine("进入框内中间 左键松开"); + } + } + else + { + if (_prevMouseEvent != MOUSEEVENTF.MOUSEEVENTF_LEFTDOWN) + { + simulator.Mouse.LeftButtonDown(); + _prevMouseEvent = MOUSEEVENTF.MOUSEEVENTF_LEFTDOWN; + Debug.WriteLine("未到框内中间 左键按下"); + } + } + } + else + { + PutRects(new Rect(), new Rect(), new Rect()); + } + } + else + { + PutRects(new Rect(), new Rect(), new Rect()); + _noRectsCount++; + // 2s 没有矩形视为已经完成钓鱼 + if (_noRectsCount >= content.FrameRate * 2 && _prevMouseEvent != 0x0) + { + _findFishBoxTips = false; + _isFishingProcess = false; + _prevMouseEvent = 0x0; + _logger.LogInformation(" 钓鱼结束"); + //_logger.LogInformation(@"└------------------------┘"); + } + } + } + + private void PutRects(Rect left, Rect cur, Rect right) + { + var list = new List<(string, System.Windows.Rect)> + { + ("FishingBarLeft", left.ToWindowsRectangleOffset(_fishBoxRect.X, _fishBoxRect.Y)), + ("FishingBarCur", cur.ToWindowsRectangleOffset(_fishBoxRect.X, _fishBoxRect.Y)), + ("FishingBarRight", right.ToWindowsRectangleOffset(_fishBoxRect.X, _fishBoxRect.Y)) + }; + VisionContext.Instance().DrawContent.PutOrRemoveRectList(list); + } + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs b/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs index 403beb1b..7c9d1916 100644 --- a/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs +++ b/BetterGenshinImpact/GameTask/AutoSkip/AutoSkipTrigger.cs @@ -13,6 +13,9 @@ using Point = OpenCvSharp.Point; namespace BetterGenshinImpact.GameTask.AutoSkip { + /// + /// 自动剧情有选项点击，必须使用BitBlt + /// public class AutoSkipTrigger : ITaskTrigger { private readonly ILogger _logger = App.GetLogger(); diff --git a/BetterGenshinImpact/GameTask/GameTaskManager.cs b/BetterGenshinImpact/GameTask/GameTaskManager.cs index cb1cb906..2782ac70 100644 --- a/BetterGenshinImpact/GameTask/GameTaskManager.cs +++ b/BetterGenshinImpact/GameTask/GameTaskManager.cs @@ -30,8 +30,11 @@ namespace BetterGenshinImpact.GameTask // .ForEach(type => LoadedTriggers.Add(type.CreateInstance())); // }); - List loadedTriggers = new(); - loadedTriggers.Add(new AutoSkip.AutoSkipTrigger()); + List loadedTriggers = new() + { + new AutoSkip.AutoSkipTrigger(), + new AutoFishing.AutoFishingTrigger() + }; loadedTriggers.ForEach(i => i.Init()); diff --git a/BetterGenshinImpact/GameTask/TaskDispatcher.cs b/BetterGenshinImpact/GameTask/TaskDispatcher.cs index 4cad6d50..cba6fd22 100644 --- a/BetterGenshinImpact/GameTask/TaskDispatcher.cs +++ b/BetterGenshinImpact/GameTask/TaskDispatcher.cs @@ -39,7 +39,7 @@ namespace BetterGenshinImpact.GameTask _timer.Elapsed += Tick; } - public void Start(CaptureMode mode, int frameRate = 30) + public void Start(CaptureMode mode, int frameRate = 60) { IntPtr hWnd = SystemControl.FindGenshinImpactHandle(); if (hWnd == IntPtr.Zero) @@ -98,7 +98,7 @@ namespace BetterGenshinImpact.GameTask } // 循环执行所有触发器 有独占状态的触发器的时候只执行独占触发器 - var content = new CaptureContent(bitmap, _frameIndex); + var content = new CaptureContent(bitmap, _frameIndex, _frameRate); var exclusiveTrigger = _triggers.FirstOrDefault(t => t is { IsEnabled: true, IsExclusive: true }); if (exclusiveTrigger != null) { diff --git a/BetterGenshinImpact/View/MainWindow.xaml b/BetterGenshinImpact/View/MainWindow.xaml index 6d4cd839..4ff631f9 100644 --- a/BetterGenshinImpact/View/MainWindow.xaml +++ b/BetterGenshinImpact/View/MainWindow.xaml @@ -6,7 +6,7 @@ xmlns:b="http://schemas.microsoft.com/xaml/behaviors" xmlns:viewModel="clr-namespace:BetterGenshinImpact.ViewModel" mc:Ignorable="d" - Title="MainWindow" Height="450" Width="200"> + Title="MainWindow" Height="200" Width="300"> diff --git a/Vision.Recognition/DrawContent.cs b/Vision.Recognition/DrawContent.cs index 496091a5..5ca18764 100644 --- a/Vision.Recognition/DrawContent.cs +++ b/Vision.Recognition/DrawContent.cs @@ -1,9 +1,11 @@ -using System; + using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Security.Policy; using System.Text; using System.Threading.Tasks; +using System.Windows.Input; namespace Vision.Recognition { @@ -33,6 +35,41 @@ namespace Vision.Recognition MaskWindow.Instance().Refresh(); } + public void PutOrRemoveRectList(List<(string, System.Windows.Rect)> list) + { + bool changed = false; + list.ForEach(item => + { + var newRect = item.Item2; + if (newRect.IsEmpty) + { + if (RectList.TryGetValue(item.Item1, out _)) + { + RectList.TryRemove(item.Item1, out _); + changed = true; + } + } + else + { + if (RectList.TryGetValue(item.Item1, out var prevRect)) + { + if (newRect == prevRect) + { + return; + } + } + RectList[item.Item1] = newRect; + changed = true; + } + + + }); + if (changed) + { + MaskWindow.Instance().Refresh(); + } + } + public void RemoveRect(string key) { if (RectList.TryGetValue(key, out _)) diff --git a/Vision.Recognition/Helper/OpenCv/CommonExtension.cs b/Vision.Recognition/Helper/OpenCv/CommonExtension.cs index 49cd6110..26bc4a97 100644 --- a/Vision.Recognition/Helper/OpenCv/CommonExtension.cs +++ b/Vision.Recognition/Helper/OpenCv/CommonExtension.cs @@ -34,6 +34,11 @@ namespace Vision.Recognition.Helper.OpenCv return new System.Windows.Rect(rect.X, rect.Y, rect.Width, rect.Height); } + public static System.Windows.Rect ToWindowsRectangleOffset(this OpenCvSharp.Rect rect, int offsetX, int offsetY) + { + return new System.Windows.Rect(rect.X + offsetX, rect.Y + offsetY, rect.Width, rect.Height); + } + public static Rectangle ToDrawingRectangle(this OpenCvSharp.Rect rect) { return new Rectangle(rect.X, rect.Y, rect.Width, rect.Height); diff --git a/Vision.Recognition/MaskWindow.xaml.cs b/Vision.Recognition/MaskWindow.xaml.cs index 92c511f6..f2d9f00f 100644 --- a/Vision.Recognition/MaskWindow.xaml.cs +++ b/Vision.Recognition/MaskWindow.xaml.cs @@ -69,16 +69,23 @@ namespace Vision.Recognition { foreach (var kv in VisionContext.Instance().DrawContent.RectList) { - drawingContext.DrawRectangle(Brushes.Transparent, new Pen(Brushes.Red, 2), kv.Value); + if (!kv.Value.IsEmpty) + { + drawingContext.DrawRectangle(Brushes.Transparent, new Pen(Brushes.Red, 2), kv.Value); + } + } foreach (var kv in VisionContext.Instance().DrawContent.TextList) { - drawingContext.DrawText(new FormattedText(kv.Value.Item2, - CultureInfo.GetCultureInfo("zh-cn"), - FlowDirection.LeftToRight, - MyTypeface, - 36, Brushes.Black, 1), kv.Value.Item1); + if (kv.Value.Item1.X != 0 || kv.Value.Item1.Y != 0) + { + drawingContext.DrawText(new FormattedText(kv.Value.Item2, + CultureInfo.GetCultureInfo("zh-cn"), + FlowDirection.LeftToRight, + MyTypeface, + 36, Brushes.Black, 1), kv.Value.Item1); + } } } catch (Exception e) diff --git a/Vision.Recognition/Task/CaptureContent.cs b/Vision.Recognition/Task/CaptureContent.cs index af48f9da..421f13af 100644 --- a/Vision.Recognition/Task/CaptureContent.cs +++ b/Vision.Recognition/Task/CaptureContent.cs @@ -17,11 +17,13 @@ namespace Vision.Recognition.Task { public Bitmap SrcBitmap { get; } public int FrameIndex { get; private set; } + public int FrameRate { get; private set; } - public CaptureContent(Bitmap srcBitmap, int frameIndex) + public CaptureContent(Bitmap srcBitmap, int frameIndex, int frameRate) { SrcBitmap = srcBitmap; FrameIndex = frameIndex; + FrameRate = frameRate; } private Mat? _srcMat;