diff --git a/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs b/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs index 23b3d5e2..68dd79d7 100644 --- a/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs +++ b/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs @@ -24,6 +24,7 @@ using BetterGenshinImpact.Core.Recognition.OpenCv; using BetterGenshinImpact.Core.Recognition.OCR; using BetterGenshinImpact.GameTask.Common; using BetterGenshinImpact.GameTask.Common.Job; +using GameTask.Model.GameUI; namespace BetterGenshinImpact.GameTask.AutoArtifactSalvage; @@ -210,118 +211,42 @@ public class AutoArtifactSalvageTask : ISoloTask private async Task Salvage5Star(string regularExpression, int maxNumToCheck) { int count = maxNumToCheck; - Queue checkedArtifactAffixesQueue = new Queue(); - int duplicateSum = 0; - while (count > 0 && duplicateSum < 3) + + using var ra0 = CaptureToRectArea(); + Rect gridRoi = new Rect((int)(ra0.Width * 0.025), (int)(ra0.Width * 0.055), (int)(ra0.Width * 0.66), (int)(ra0.Width * 0.4)); + GridScreen gridScreen = new GridScreen(gridRoi, 3, 40, 28, 0.018, this.logger, this.ct); // 圣遗物分解Grid有4行9列 + await foreach (ImageRegion itemRegion in gridScreen) { - // VisionContext.Instance().DrawContent.ClearAll(); - // await Delay(400, this.ct); - - using var ra = CaptureToRectArea(); - using ImageRegion grid = ra.DeriveCrop(new Rect((int)(ra.Width * 0.025), (int)(ra.Width * 0.055), (int)(ra.Width * 0.66), (int)(ra.Width * 0.4))); - IEnumerable gridItems = GetArtifactGridItems(grid.SrcMat); - - //foreach (Rect item in gridItems) - //{ - // grid.DrawRect(item, item.GetHashCode().ToString(), new System.Drawing.Pen(System.Drawing.Color.Blue)); - //} - - bool anyItemChecked = false; - foreach (Rect item in gridItems) + Rect gridRect = itemRegion.ToRect(); + if (GetArtifactStatus(itemRegion.SrcMat) == ArtifactStatus.None) { - using ImageRegion itemRegion = grid.DeriveCrop(item); - if (GetArtifactStatus(itemRegion.SrcMat) == ArtifactStatus.None) + itemRegion.Click(); + await Delay(300, ct); + + using var ra1 = CaptureToRectArea(); + using ImageRegion itemRegion1 = ra1.DeriveCrop(gridRect + new Point(gridRoi.X, gridRoi.Y)); + if (GetArtifactStatus(itemRegion1.SrcMat) == ArtifactStatus.Selected) { - anyItemChecked = true; - itemRegion.Click(); - await Delay(300, ct); + using ImageRegion card = ra1.DeriveCrop(new Rect((int)(ra1.Width * 0.70), (int)(ra1.Width * 0.055), (int)(ra1.Width * 0.24), (int)(ra1.Width * 0.29))); + string affixes = GetArtifactAffixes(card.SrcMat, OcrFactory.Paddle); - using var ra1 = CaptureToRectArea(); - using ImageRegion grid1 = ra1.DeriveCrop(new Rect((int)(ra1.Width * 0.025), (int)(ra1.Width * 0.055), (int)(ra1.Width * 0.66), (int)(ra1.Width * 0.4))); - using ImageRegion itemRegion1 = grid1.DeriveCrop(item); - if (GetArtifactStatus(itemRegion1.SrcMat) == ArtifactStatus.Selected) + if (IsMatchRegularExpression(affixes, regularExpression, out string msg)) { - using ImageRegion card = ra1.DeriveCrop(new Rect((int)(ra1.Width * 0.70), (int)(ra1.Width * 0.055), (int)(ra1.Width * 0.24), (int)(ra1.Width * 0.29))); - string affixes = GetArtifactAffixes(card.SrcMat, OcrFactory.Paddle); - - if (checkedArtifactAffixesQueue.Any(c => c == affixes)) - { - duplicateSum++; - logger.LogInformation($"重复检查了该圣遗物"); - } - if (checkedArtifactAffixesQueue.Count >= 36) // 一个grid最多能看到36个完整的圣遗物 - { - checkedArtifactAffixesQueue.Dequeue(); - } - checkedArtifactAffixesQueue.Enqueue(affixes); - - if (IsMatchRegularExpression(affixes, regularExpression, out string msg)) - { - logger.LogInformation(message: msg); - } - else - { - itemRegion.Click(); - await Delay(100, ct); - } - if (duplicateSum >= 3) - { - break; - } - } - count--; - if (count <= 0) - { - break; - } - } - } - if (count <= 0 || duplicateSum >= 3) - { - break; - } - if (anyItemChecked) - { - for (int i = 0; i < 32; i++) // 先滚动大约三行半 - { - input.Mouse.VerticalScroll(-2); - await Delay(40, ct); - } - - DateTimeOffset rollingEndTime = DateTime.Now.AddSeconds(2); - while (DateTime.Now < rollingEndTime) - { - await Delay(60, ct); - using var ra2 = CaptureToRectArea(); - using ImageRegion grid2 = ra2.DeriveCrop(new Rect((int)(ra2.Width * 0.025), (int)(ra2.Width * 0.055), (int)(ra2.Width * 0.66), (int)(ra2.Width * 0.4))); - IEnumerable gridItems2 = GetArtifactGridItems(grid2.SrcMat); - if (gridItems2.Min(i => i.Y) > (ra2.Width * 0.018)) // 精细滚动，保证完整地显示四行 - { - input.Mouse.VerticalScroll(-1); + logger.LogInformation(message: msg); } else { - break; + itemRegion.Click(); + await Delay(100, ct); } } - - grid.MoveTo(grid.Width, grid.Height); - await Delay(500, ct); + count--; + if (count <= 0) + { + logger.LogInformation("检查次数已耗尽"); + break; + } } - else - { - await Delay(400, ct); - logger.LogInformation("找不到可检查的圣遗物了"); - break; - } - } - if (count <= 0) - { - logger.LogInformation("检查次数已耗尽"); - } - if (duplicateSum >= 3) - { - logger.LogInformation("重复检查次数过多，推断为找不到可检查的了"); } } @@ -352,33 +277,6 @@ public class AutoArtifactSalvageTask : ISoloTask return ocrResult.Text; } - public static IEnumerable GetArtifactGridItems(Mat src) - { - using Mat grey = src.CvtColor(ColorConversionCodes.BGR2GRAY); - - using Mat canny = grey.Canny(20, 40); - - Cv2.FindContours(canny, out var contours, out _, RetrievalModes.External, - ContourApproximationModes.ApproxSimple, null); - - IEnumerable boxes = contours.Where(c => Cv2.MinAreaRect(c).Angle % 90 <= 1) // 剔除倾斜 - .Select(Cv2.BoundingRect).Where(r => - { - if (r.Height == 0) - { - return false; - } - return Math.Abs((float)r.Width / r.Height - 0.8) < 0.05; // 按形状筛选 - }).ToList(); - - //src.DrawContours(contours, -1, Scalar.Red); - - int biggestRectHeight = boxes.Max(b => b.Height); - boxes = boxes.Where(b => (float)b.Height / biggestRectHeight > 0.88); // 剔除太小的 - - return boxes.ToArray(); - } - public static ArtifactStatus GetArtifactStatus(Mat src) { using Mat upperLine = new Mat(src, new Rect(0, 0, src.Width, (int)(src.Height * 0.19))); diff --git a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs new file mode 100644 index 00000000..5b24d52f --- /dev/null +++ b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs @@ -0,0 +1,232 @@ +using BetterGenshinImpact.Core.Simulator; +using BetterGenshinImpact.GameTask.Common; +using BetterGenshinImpact.GameTask.Model.Area; +using Fischless.WindowsInput; +using Microsoft.Extensions.Logging; +using OpenCvSharp; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace GameTask.Model.GameUI +{ + public class GridScreen : IAsyncEnumerable + { + private readonly Rect gridRoi; + private readonly CancellationToken ct; + private readonly ILogger logger; + private readonly InputSimulator input = Simulation.SendInput; + private readonly int s1Round; + private readonly int roundMilliseconds; + private readonly int s2Round; + private readonly double s3Scale; + + /// + /// 对Gird类型界面的操作封装类 + /// 直接对此类对象进行遍历即可获取所有项 + /// 每次的截图是上次滚动后的，如果实时性要求高，应每次迭代自行截图 + /// 在末页可能重复返回GridItem，须自行处理 + /// + /// Grid所在位置 + /// + /// + public GridScreen(Rect gridRoi, int s1Round, int roundMilliseconds, int s2Round, double s3Scale, ILogger logger, CancellationToken ct) + { + this.gridRoi = gridRoi; + this.ct = ct; + this.logger = logger; + this.s1Round = s1Round; + this.roundMilliseconds = roundMilliseconds; + this.s2Round = s2Round; + this.s3Scale = s3Scale; + } + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + return new GridEnumerator(this.gridRoi, this.s1Round, this.roundMilliseconds, this.s2Round, this.s3Scale, this.logger, this.input, this.ct); + } + + public class GridEnumerator : IAsyncEnumerator + { + private readonly Rect roi; + private readonly CancellationToken ct; + private readonly ILogger logger; + private readonly InputSimulator input = Simulation.SendInput; + private readonly int s1Round; + private readonly int roundMilliseconds; + private readonly int s2Round; + private readonly double s3Scale; + + private record Page(ImageRegion ImageRegion, Stack Rects); + private Page? currentPage; + private ImageRegion current; + ImageRegion IAsyncEnumerator.Current => current; + + /// + /// 滚动操作枚举器 + /// + /// + /// 测试是否能滚动时发出的滚动命令次数 + /// 滚动命令间隔毫秒 + /// 滚过一整页时发出的滚动命令次数 + /// 微调滚动时控制首行距离上边界的参数 + /// + /// + /// + public GridEnumerator(Rect roi, int s1Round, int roundMilliseconds, int s2Round, double s3Scale, ILogger logger, InputSimulator input, CancellationToken ct) + { + this.roi = roi; + this.ct = ct; + this.logger = logger; + this.input = input; + this.s1Round = s1Round; + this.roundMilliseconds = roundMilliseconds; + this.s2Round = s2Round; + this.s3Scale = s3Scale; + } + + public async Task TryVerticalScollDown() + { + using var ra = TaskControl.CaptureToRectArea(); + using ImageRegion prevGrid = ra.DeriveCrop(roi); + + for (int i = 0; i < this.s1Round; i++) + { + this.input.Mouse.VerticalScroll(-2); + await TaskControl.Delay(this.roundMilliseconds, this.ct); + } + await TaskControl.Delay(300, this.ct); + using var ra2 = TaskControl.CaptureToRectArea(); + using ImageRegion scrolledGrid = ra2.DeriveCrop(this.roi); + + bool isScrolling = IsScrolling(prevGrid.CacheGreyMat, scrolledGrid.CacheGreyMat, out Point2d shift, logger: this.logger); + + return isScrolling; + } + + /// + /// 判断是否还能继续滚动，如果到底了则只能滚动一丝并很快地回弹 + /// + /// 先前的灰度图 + /// 尝试滚动并等待可能的回弹后的灰度图 + /// 估计的位移 + /// 低于下限则可能不存在平移 + /// 上限用于抵消微小的其他差异 + /// + /// + public static bool IsScrolling(Mat prevGray, Mat nextGray, out Point2d shift, double lowerThreshold = 0.5, double upperThreshold = 0.95, ILogger? logger = null) + { + using Mat prev = new Mat(); + prevGray.ConvertTo(prev, MatType.CV_32FC1); + using Mat next = new Mat(); + nextGray.ConvertTo(next, MatType.CV_32FC1); + + using Mat window = new Mat(); + shift = Cv2.PhaseCorrelate(prev, next, window, out double response); // 相位相关性 + logger?.LogInformation($"response={response:F3}, shift=({shift.X:F2}, {shift.Y:F2})"); + return response > lowerThreshold && response < upperThreshold; + } + + public static IEnumerable GetGridItems(Mat src) + { + using Mat grey = src.CvtColor(ColorConversionCodes.BGR2GRAY); + + using Mat canny = grey.Canny(20, 40); + + Cv2.FindContours(canny, out var contours, out _, RetrievalModes.External, ContourApproximationModes.ApproxSimple, null); + + IEnumerable boxes = contours.Where(c => Cv2.MinAreaRect(c).Angle % 90 <= 1) // 剔除倾斜 + .Select(Cv2.BoundingRect).Where(r => + { + if (r.Height == 0) + { + return false; + } + return Math.Abs((float)r.Width / r.Height - 0.8) < 0.05; // 按形状筛选 + }).ToList(); + + //src.DrawContours(contours, -1, Scalar.Red); + + int biggestRectHeight = boxes.Max(b => b.Height); + boxes = boxes.Where(b => (float)b.Height / biggestRectHeight > 0.88); // 剔除太小的 + + return boxes.ToArray(); + } + + public async ValueTask MoveNextAsync() + { + if (this.currentPage == null || this.currentPage.Rects.Count < 1) + { + if (this.currentPage != null) + { + //BetterGenshinImpact.View.Drawable.VisionContext.Instance().DrawContent.ClearAll(); + + using var ra4 = TaskControl.CaptureToRectArea(); + ra4.MoveTo(this.roi.X + this.roi.Width / 2, this.roi.Y + this.roi.Height / 2); + await TaskControl.Delay(300, ct); + + bool canScoll = await TryVerticalScollDown(); + + if (canScoll) + { + for (int i = 0; i < this.s2Round; i++) // 再滚动差不多（最多行数-1）行 + { + input.Mouse.VerticalScroll(-2); + await TaskControl.Delay(this.roundMilliseconds, ct); + } + + DateTimeOffset rollingEndTime = DateTime.Now.AddSeconds(2); + while (DateTime.Now < rollingEndTime) + { + await TaskControl.Delay(60, ct); + using var ra2 = TaskControl.CaptureToRectArea(); + using ImageRegion grid2 = ra2.DeriveCrop(this.roi); + IEnumerable gridItems2 = GetGridItems(grid2.SrcMat); + if (gridItems2.Min(i => i.Y) > (ra2.Width * this.s3Scale)) // 最后精细滚动，保证完整地显示最多行 + { + input.Mouse.VerticalScroll(-1); + } + else + { + break; + } + } + using var ra3 = TaskControl.CaptureToRectArea(); + using ImageRegion grid3 = ra3.DeriveCrop(this.roi); + grid3.MoveTo(grid3.Width, grid3.Height); + await TaskControl.Delay(300, ct); + } + else + { + await TaskControl.Delay(300, ct); + this.logger.LogInformation("滚动到底部了"); + return false; + } + } + + using var ra = TaskControl.CaptureToRectArea(); + var imageRegion = ra.DeriveCrop(this.roi); + IEnumerable gridItems = GetGridItems(imageRegion.SrcMat); + this.currentPage = new Page(imageRegion, new Stack(gridItems)); + + //foreach (Rect item in gridItems) + //{ + // imageRegion.DrawRect(item, item.GetHashCode().ToString(), new System.Drawing.Pen(System.Drawing.Color.Blue)); + //} + } + + this.current = this.currentPage.ImageRegion.DeriveCrop(this.currentPage.Rects.Pop()); + return true; + } + + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + } + } +} diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs index 7f67f71a..7f01e39e 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs @@ -1,5 +1,6 @@ using BetterGenshinImpact.GameTask.AutoArtifactSalvage; using BetterGenshinImpact.UnitTest.CoreTests.RecognitionTests.OCRTests; +using GameTask.Model.GameUI; using OpenCvSharp; using System; using System.Collections.Concurrent; @@ -52,7 +53,7 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoArtifactSalvageTests using Mat mat = new Mat(@$"..\..\..\Assets\AutoArtifactSalvage\{screenshot}"); // - var result = AutoArtifactSalvageTask.GetArtifactGridItems(mat); + var result = GridScreen.GridEnumerator.GetGridItems(mat); // Assert.Equal(4, result.Count()); @@ -69,7 +70,7 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoArtifactSalvageTests using Mat mat = new Mat(@$"..\..\..\Assets\AutoArtifactSalvage\ArtifactGrid.png"); // - var result = AutoArtifactSalvageTask.GetArtifactGridItems(mat); + var result = GridScreen.GridEnumerator.GetGridItems(mat); using Mat leftTopOne = new Mat(mat, result.Single(r => r.X + r.Width / 2 < mat.Width / 2 && r.Y + r.Height / 2 < mat.Height / 2)); using Mat rightTopOne = new Mat(mat, result.Single(r => r.X + r.Width / 2 > mat.Width / 2 && r.Y + r.Height / 2 < mat.Height / 2)); using Mat leftBottomOne = new Mat(mat, result.Single(r => r.X + r.Width / 2 < mat.Width / 2 && r.Y + r.Height / 2 > mat.Height / 2)); diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs new file mode 100644 index 00000000..0d599679 --- /dev/null +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs @@ -0,0 +1,39 @@ +using GameTask.Model.GameUI; +using OpenCvSharp; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BetterGenshinImpact.UnitTest.GameTaskTests.Model.GameUI +{ + public class GridScreenTests + { + [Fact] + /// + /// 测试判断前后两张图是否属于滚动，结果应正确 + /// + public void IsScrolling_ResultShouldBeRight() + { + // + using Mat mat = new Mat(@$"..\..\..\Assets\AutoArtifactSalvage\ArtifactGrid.png", flags: ImreadModes.Grayscale); + using Mat cropped = mat[new Rect(0, 0, mat.Width, mat.Height - 10)]; + using Mat black = new Mat(mat.Size(), mat.Type(), Scalar.Black); + using Mat scrolled = black.Clone(); // 一个向下平移了10像素的图 + using Mat pos = scrolled[new Rect(0, 10, mat.Width, mat.Height - 10)]; + cropped.CopyTo(pos); + + // + bool result1 = GridScreen.GridEnumerator.IsScrolling(mat, scrolled, out Point2d shift); + bool result2 = GridScreen.GridEnumerator.IsScrolling(mat, mat, out Point2d _); + bool result3 = GridScreen.GridEnumerator.IsScrolling(mat, black, out Point2d _); + + // + Assert.True(result1); + Assert.True(shift.Y <= 10 && shift.Y > 9.9); + Assert.False(result2); + Assert.False(result3); + } + } +}