using BetterGenshinImpact.Core.Simulator; using BetterGenshinImpact.GameTask.Common; using BetterGenshinImpact.GameTask.Model.Area; using Fischless.WindowsInput; using Microsoft.Extensions.Logging; using OpenCvSharp; using System; using System.Collections; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; namespace GameTask.Model.GameUI { public class GridScreen : IAsyncEnumerable { private readonly Rect gridRoi; private readonly CancellationToken ct; private readonly ILogger logger; private readonly InputSimulator input = Simulation.SendInput; private readonly int s1Round; private readonly int roundMilliseconds; private readonly int s2Round; private readonly double s3Scale; /// /// 对Gird类型界面的操作封装类 /// 直接对此类对象进行遍历即可获取所有项 /// 每次的截图是上次滚动后的,如果实时性要求高,应每次迭代自行截图 /// 在末页可能重复返回GridItem,须自行处理 /// /// Grid所在位置 /// /// public GridScreen(Rect gridRoi, int s1Round, int roundMilliseconds, int s2Round, double s3Scale, ILogger logger, CancellationToken ct) { this.gridRoi = gridRoi; this.ct = ct; this.logger = logger; this.s1Round = s1Round; this.roundMilliseconds = roundMilliseconds; this.s2Round = s2Round; this.s3Scale = s3Scale; } public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { return new GridEnumerator(this.gridRoi, this.s1Round, this.roundMilliseconds, this.s2Round, this.s3Scale, this.logger, this.input, this.ct); } public class GridEnumerator : IAsyncEnumerator { private readonly Rect roi; private readonly CancellationToken ct; private readonly ILogger logger; private readonly InputSimulator input = Simulation.SendInput; private readonly int s1Round; private readonly int roundMilliseconds; private readonly int s2Round; private readonly double s3Scale; private record Page(ImageRegion ImageRegion, Stack Rects); private Page? currentPage; private ImageRegion current; ImageRegion IAsyncEnumerator.Current => current; /// /// 滚动操作枚举器 /// /// /// 测试是否能滚动时发出的滚动命令次数 /// 滚动命令间隔毫秒 /// 滚过一整页时发出的滚动命令次数 /// 微调滚动时控制首行距离上边界的参数 /// /// /// public GridEnumerator(Rect roi, int s1Round, int roundMilliseconds, int s2Round, double s3Scale, ILogger logger, InputSimulator input, CancellationToken ct) { this.roi = roi; this.ct = ct; this.logger = logger; this.input = input; this.s1Round = s1Round; this.roundMilliseconds = roundMilliseconds; this.s2Round = s2Round; this.s3Scale = s3Scale; } public async Task TryVerticalScollDown() { using var ra = TaskControl.CaptureToRectArea(); using ImageRegion prevGrid = ra.DeriveCrop(roi); for (int i = 0; i < this.s1Round; i++) { this.input.Mouse.VerticalScroll(-2); await TaskControl.Delay(this.roundMilliseconds, this.ct); } await TaskControl.Delay(300, this.ct); using var ra2 = TaskControl.CaptureToRectArea(); using ImageRegion scrolledGrid = ra2.DeriveCrop(this.roi); bool isScrolling = IsScrolling(prevGrid.CacheGreyMat, scrolledGrid.CacheGreyMat, out Point2d shift, logger: this.logger); return isScrolling; } /// /// 判断是否还能继续滚动,如果到底了则只能滚动一丝并很快地回弹 /// /// 先前的灰度图 /// 尝试滚动并等待可能的回弹后的灰度图 /// 估计的位移 /// 低于下限则可能不存在平移 /// 上限用于抵消微小的其他差异 /// /// public static bool IsScrolling(Mat prevGray, Mat nextGray, out Point2d shift, double lowerThreshold = 0.5, double upperThreshold = 0.95, ILogger? logger = null) { using Mat prev = new Mat(); prevGray.ConvertTo(prev, MatType.CV_32FC1); using Mat next = new Mat(); nextGray.ConvertTo(next, MatType.CV_32FC1); using Mat window = new Mat(); shift = Cv2.PhaseCorrelate(prev, next, window, out double response); // 相位相关性 logger?.LogInformation($"response={response:F3}, shift=({shift.X:F2}, {shift.Y:F2})"); return response > lowerThreshold && response < upperThreshold; } public static IEnumerable GetGridItems(Mat src) { using Mat grey = src.CvtColor(ColorConversionCodes.BGR2GRAY); using Mat canny = grey.Canny(20, 40); Cv2.FindContours(canny, out var contours, out _, RetrievalModes.External, ContourApproximationModes.ApproxSimple, null); IEnumerable boxes = contours.Where(c => Cv2.MinAreaRect(c).Angle % 90 <= 1) // 剔除倾斜 .Select(Cv2.BoundingRect).Where(r => { if (r.Height == 0) { return false; } return Math.Abs((float)r.Width / r.Height - 0.8) < 0.05; // 按形状筛选 }).ToList(); //src.DrawContours(contours, -1, Scalar.Red); int biggestRectHeight = boxes.Max(b => b.Height); boxes = boxes.Where(b => (float)b.Height / biggestRectHeight > 0.88); // 剔除太小的 return boxes.ToArray(); } public async ValueTask MoveNextAsync() { if (this.currentPage == null || this.currentPage.Rects.Count < 1) { if (this.currentPage != null) { //BetterGenshinImpact.View.Drawable.VisionContext.Instance().DrawContent.ClearAll(); using var ra4 = TaskControl.CaptureToRectArea(); ra4.MoveTo(this.roi.X + this.roi.Width / 2, this.roi.Y + this.roi.Height / 2); await TaskControl.Delay(300, ct); bool canScoll = await TryVerticalScollDown(); if (canScoll) { for (int i = 0; i < this.s2Round; i++) // 再滚动差不多(最多行数-1)行 { input.Mouse.VerticalScroll(-2); await TaskControl.Delay(this.roundMilliseconds, ct); } DateTimeOffset rollingEndTime = DateTime.Now.AddSeconds(2); while (DateTime.Now < rollingEndTime) { await TaskControl.Delay(60, ct); using var ra2 = TaskControl.CaptureToRectArea(); using ImageRegion grid2 = ra2.DeriveCrop(this.roi); IEnumerable gridItems2 = GetGridItems(grid2.SrcMat); if (gridItems2.Min(i => i.Y) > (ra2.Width * this.s3Scale)) // 最后精细滚动,保证完整地显示最多行 { input.Mouse.VerticalScroll(-1); } else { break; } } using var ra3 = TaskControl.CaptureToRectArea(); using ImageRegion grid3 = ra3.DeriveCrop(this.roi); grid3.MoveTo(grid3.Width, grid3.Height); await TaskControl.Delay(300, ct); } else { await TaskControl.Delay(300, ct); this.logger.LogInformation("滚动到底部了"); return false; } } using var ra = TaskControl.CaptureToRectArea(); var imageRegion = ra.DeriveCrop(this.roi); IEnumerable gridItems = GetGridItems(imageRegion.SrcMat); this.currentPage = new Page(imageRegion, new Stack(gridItems)); //foreach (Rect item in gridItems) //{ // imageRegion.DrawRect(item, item.GetHashCode().ToString(), new System.Drawing.Pen(System.Drawing.Color.Blue)); //} } this.current = this.currentPage.ImageRegion.DeriveCrop(this.currentPage.Rects.Pop()); return true; } public ValueTask DisposeAsync() { return ValueTask.CompletedTask; } } } }