From bc346d1680e9dd54f9114d4933302a9a3d4a60d9 Mon Sep 17 00:00:00 2001 From: FishmanTheMurloc <162452111+FishmanTheMurloc@users.noreply.github.com> Date: Sat, 1 Nov 2025 14:12:41 +0800 Subject: [PATCH] =?UTF-8?q?GridScreen=E5=A2=9E=E5=8A=A0=E5=A1=AB=E5=85=85?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E4=BB=A5=E5=BA=94=E5=AF=B9item=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E4=B8=8D=E5=85=A8=E7=9A=84=E9=97=AE=E9=A2=98=20(#2423?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AutoArtifactSalvageTask.cs | 26 +- .../GameTask/AutoEat/AutoEatTask.cs | 3 +- .../GameTask/AutoFishing/AutoFishingTask.cs | 2 +- .../GameTask/AutoFishing/Behaviours.cs | 9 +- .../GameTask/Common/Job/CountInventoryItem.cs | 6 +- .../GameTask/GetGridIcons/GetGridIconsTask.cs | 10 +- .../GetGridIcons/GridIconsAccuracyTestTask.cs | 3 +- .../Model/GameUI/ArtifactSetFilterScreen.cs | 170 +++-------- .../GameTask/Model/GameUI/GridCell.cs | 162 +++++++++++ .../GameTask/Model/GameUI/GridScreen.cs | 272 +++++++++--------- .../Model/GameUI/GridScreenExtensions.cs | 15 +- .../AutoArtifactSalvageTaskTests.cs | 10 +- .../GridIconsAccuracyTestTaskTests.cs | 15 +- .../Model/GameUI/GridScreenTests.cs | 30 +- 14 files changed, 407 insertions(+), 326 deletions(-) create mode 100644 BetterGenshinImpact/GameTask/Model/GameUI/GridCell.cs diff --git a/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs b/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs index d9324975..09f8c3a7 100644 --- a/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs +++ b/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs @@ -315,8 +315,9 @@ public class AutoArtifactSalvageTask : ISoloTask gridScreen.OnBeforeScroll += () => { VisionContext.Instance().DrawContent.RemoveRect(drawKey); drawRectList.Clear(); drawTextList.Clear(); }; try { - await foreach (ImageRegion itemRegion in gridScreen) + await foreach ((ImageRegion pageRegion, Rect itemRect) in gridScreen) { + using ImageRegion itemRegion = pageRegion.DeriveCrop(itemRect); using Mat img125 = GetGridIconsTask.CropResizeArtifactSetFilterGridIcon(itemRegion); (string? predName, _) = GridIconsAccuracyTestTask.Infer(img125, session, prototypes); if (predName == null) @@ -383,8 +384,9 @@ public class AutoArtifactSalvageTask : ISoloTask gridScreen.OnBeforeScroll += () => VisionContext.Instance().DrawContent.ClearAll(); try { - await foreach (ImageRegion itemRegion in gridScreen) + await foreach ((ImageRegion pageRegion, Rect itemRect) in gridScreen) { + using ImageRegion itemRegion = pageRegion.DeriveCrop(itemRect); Rect gridRect = itemRegion.ToRect(); if (GetArtifactStatus(itemRegion.SrcMat) == ArtifactStatus.None) { @@ -526,25 +528,25 @@ public class AutoArtifactSalvageTask : ISoloTask public ArtifactStat GetArtifactStat(Mat src, IOcrService ocrService, out string allText) { using Mat gray = src.CvtColor(ColorConversionCodes.BGR2GRAY); - Mat hatKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(15, 15)/*需根据实际文本大小调整*/); // 顶帽运算核 + using Mat hatKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(15, 15)/*需根据实际文本大小调整*/); // 顶帽运算核 - Mat nameRoi = gray.SubMat(new Rect(0, 0, src.Width, (int)(src.Height * 0.106))); + using Mat nameRoi = gray.SubMat(new Rect(0, 0, src.Width, (int)(src.Height * 0.106))); //Cv2.ImShow("name", nameRoi); - Mat typeRoi = gray.SubMat(new Rect(0, (int)(src.Height * 0.106), src.Width, (int)(src.Height * 0.106))); + using Mat typeRoi = gray.SubMat(new Rect(0, (int)(src.Height * 0.106), src.Width, (int)(src.Height * 0.106))); #region 主词条预处理 去除背景干扰 - Mat mainAffixRoi = gray.SubMat(new Rect(0, (int)(src.Height * 0.22), (int)(src.Width * 0.55), (int)(src.Height * 0.30))); + using Mat mainAffixRoi = gray.SubMat(new Rect(0, (int)(src.Height * 0.22), (int)(src.Width * 0.55), (int)(src.Height * 0.30))); using Mat mainAffixRoiBottomHat = mainAffixRoi.MorphologyEx(MorphTypes.TopHat, hatKernel); using Mat mainAffixRoiThreshold = mainAffixRoiBottomHat.Threshold(30, 255, ThresholdTypes.Binary); //Cv2.ImShow("mainAffix", mainAffixRoiThreshold); #endregion #region 副词条预处理 还是不处理效果最好…… - Mat levelAndMinorAffixRoi = gray.SubMat(new Rect(0, (int)(src.Height * 0.52), src.Width, (int)(src.Height * 0.48))); - //using Mat levelAndMinorAffixRoiThreshold = new Mat(); + using Mat levelAndMinorAffixRoi = gray.SubMat(new Rect(0, (int)(src.Height * 0.52), src.Width, (int)(src.Height * 0.48))); + //using Mat levelAndMinorAffixRoiThreshold = new Mat(); // otsu确定阈值大概在170 //double otsu = Cv2.Threshold(levelAndMinorAffixRoi, levelAndMinorAffixRoiThreshold, 0, 255, ThresholdTypes.Binary | ThresholdTypes.Otsu); - // //using Mat levelAndMinorAffixRoiThreshold = levelAndMinorAffixRoi.Threshold(170, 255, ThresholdTypes.Binary); //Cv2.ImShow($"levelAndMinorAffixRoi = {otsu}", levelAndMinorAffixRoiThreshold); - #endregion //Cv2.WaitKey(); + //using Mat levelAndMinorAffixRoiThreshold = levelAndMinorAffixRoi.Threshold(170, 255, ThresholdTypes.Binary); + #endregion var nameOcrResult = ocrService.OcrResult(nameRoi); var typeOcrResult = ocrService.OcrResult(typeRoi); @@ -607,7 +609,7 @@ public class AutoArtifactSalvageTask : ISoloTask #region 副词条 var minorAffixes = new List(); - string pattern = @"^([^+:：]+)\+([\d., ]*)(%?).*$"; + string pattern = @"^([^+:：]+)\+([\d., 。]*)(%?).*$"; pattern = pattern.Replace("%", percentStr); foreach (var r in levelAndMinorAffixResult) { @@ -672,7 +674,7 @@ public class AutoArtifactSalvageTask : ISoloTask throw new Exception($"未识别的副词条：{match.Groups[1].Value}"); } - if (!float.TryParse(match.Groups[2].Value, NumberStyles.Any, cultureInfo, out float affixValue)) + if (!float.TryParse(match.Groups[2].Value.Replace("。", "."), NumberStyles.Any, cultureInfo, out float affixValue)) { throw new Exception($"未识别的副词条数值：{match.Groups[2].Value}"); } diff --git a/BetterGenshinImpact/GameTask/AutoEat/AutoEatTask.cs b/BetterGenshinImpact/GameTask/AutoEat/AutoEatTask.cs index 47cd7200..2ff472f4 100644 --- a/BetterGenshinImpact/GameTask/AutoEat/AutoEatTask.cs +++ b/BetterGenshinImpact/GameTask/AutoEat/AutoEatTask.cs @@ -92,8 +92,9 @@ public class AutoEatTask : BaseIndependentTask, ISoloTask int? count = null; try { - await foreach (ImageRegion itemRegion in gridScreen) + await foreach ((ImageRegion pageRegion, Rect itemRect) in gridScreen) { + using ImageRegion itemRegion = pageRegion.DeriveCrop(itemRect); using Mat icon = itemRegion.SrcMat.GetGridIcon(); var result = GridIconsAccuracyTestTask.Infer(icon, session, prototypes); string predName = result.Item1; diff --git a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs index c313867b..2217a8cb 100644 --- a/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs +++ b/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs @@ -390,7 +390,7 @@ namespace BetterGenshinImpact.GameTask.AutoFishing clickWhiteConfirmButtonWaitEndTime < timeProvider.GetLocalNow()) && Bv.ClickWhiteConfirmButton(imageRegion)) { - Mat subMat = imageRegion.SrcMat.SubMat(new Rect((int)(0.824 * imageRegion.Width), (int)(0.669 * imageRegion.Height), (int)(0.065 * imageRegion.Width), (int)(0.065 * imageRegion.Width))); + using Mat subMat = imageRegion.SrcMat.SubMat(new Rect((int)(0.824 * imageRegion.Width), (int)(0.669 * imageRegion.Height), (int)(0.065 * imageRegion.Width), (int)(0.065 * imageRegion.Width))); using Mat resized = subMat.Resize(new Size(125, 125)); (string predName, _) = GridIconsAccuracyTestTask.Infer(resized, this.session, this.prototypes); if (predName.TryGetEnumValueFromDescription(out this.blackboard.selectedBait)) diff --git a/BetterGenshinImpact/GameTask/AutoFishing/Behaviours.cs b/BetterGenshinImpact/GameTask/AutoFishing/Behaviours.cs index ee49b6f3..dff8f0c8 100644 --- a/BetterGenshinImpact/GameTask/AutoFishing/Behaviours.cs +++ b/BetterGenshinImpact/GameTask/AutoFishing/Behaviours.cs @@ -18,6 +18,7 @@ using Microsoft.Extensions.Logging; using Microsoft.ML.OnnxRuntime; using OpenCvSharp; using System; +using System.Collections.Frozen; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; @@ -150,7 +151,7 @@ namespace BetterGenshinImpact.GameTask.AutoFishing // 寻找鱼饵 var boxAndBaits = FindBait(imageRegion); - ; + ; foreach ((Rect box, string? predName) in boxAndBaits) { if (predName == blackboard.selectedBait.GetDescription()) @@ -231,9 +232,15 @@ namespace BetterGenshinImpact.GameTask.AutoFishing using ImageRegion resRa = singleRowGrid.DeriveCrop(box); using Mat img125 = resRa.SrcMat.GetGridIcon(); (string? predName, _) = GridIconsAccuracyTestTask.Infer(img125, this.session, this.prototypes); + if (predName != null && !availableBaitNames.Contains(predName)) + { + predName = null; + } yield return (new Rect(singleRowGrid.X + box.X, singleRowGrid.Y + box.Y, box.Width, box.Height), predName); } } + + private static readonly FrozenSet availableBaitNames = Enum.GetValues(typeof(BaitType)).Cast().Select(bt => bt.GetDescription()).ToFrozenSet(); } [Obsolete] diff --git a/BetterGenshinImpact/GameTask/Common/Job/CountInventoryItem.cs b/BetterGenshinImpact/GameTask/Common/Job/CountInventoryItem.cs index ed418dd7..82fe8218 100644 --- a/BetterGenshinImpact/GameTask/Common/Job/CountInventoryItem.cs +++ b/BetterGenshinImpact/GameTask/Common/Job/CountInventoryItem.cs @@ -88,8 +88,9 @@ namespace BetterGenshinImpact.GameTask.Common.Job int? count = null; try { - await foreach (ImageRegion itemRegion in gridScreen) + await foreach ((ImageRegion pageRegion, Rect itemRect) in gridScreen) { + using ImageRegion itemRegion = pageRegion.DeriveCrop(itemRect); using Mat icon = itemRegion.SrcMat.GetGridIcon(); var result = GridIconsAccuracyTestTask.Infer(icon, session, prototypes); if (result.Item1 == null) @@ -135,8 +136,9 @@ namespace BetterGenshinImpact.GameTask.Common.Job gridScreen.OnBeforeScroll += () => VisionContext.Instance().DrawContent.ClearAll(); try { - await foreach (ImageRegion itemRegion in gridScreen) + await foreach ((ImageRegion pageRegion, Rect itemRect) in gridScreen) { + using ImageRegion itemRegion = pageRegion.DeriveCrop(itemRect); using Mat icon = itemRegion.SrcMat.GetGridIcon(); var result = GridIconsAccuracyTestTask.Infer(icon, session, prototypes); if (result.Item1 == null) diff --git a/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsTask.cs b/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsTask.cs index 2d515a38..3979c8bb 100644 --- a/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsTask.cs +++ b/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsTask.cs @@ -89,8 +89,9 @@ public class GetGridIconsTask : ISoloTask HashSet fileNames = new HashSet(); try { - await foreach (ImageRegion itemRegion in gridScreen) + await foreach ((ImageRegion pageRegion, Rect itemRect) in gridScreen) { + using ImageRegion itemRegion = pageRegion.DeriveCrop(itemRect); itemRegion.Click(); await Delay(300, ct); @@ -150,8 +151,9 @@ public class GetGridIconsTask : ISoloTask { ArtifactSetFilterScreen gridScreen = new ArtifactSetFilterScreen(new GridParams(new Rect(40, 100, 1300, 852), 2, 3, 40, 40, 0.024), this.logger, this.ct); HashSet fileNames = new HashSet(); - await foreach (ImageRegion itemRegion in gridScreen) + await foreach ((ImageRegion pageRegion, Rect itemRect) in gridScreen) { + using ImageRegion itemRegion = pageRegion.DeriveCrop(itemRect); itemRegion.Click(); await Delay(300, ct); @@ -175,7 +177,7 @@ public class GetGridIconsTask : ISoloTask // 截取没有符号的区域再识别一次 Rect flowerWithoutGlyph = new Rect((int)(ra1.Width * 0.028), (int)(flowerWithGlyphRect.Y - flowerWithGlyphRect.Height * 0), (int)(ra1.Width * 0.228), (int)(flowerWithGlyphRect.Height * 1)); - Mat roi = nameRegion.SrcMat.SubMat(flowerWithoutGlyph); + using Mat roi = nameRegion.SrcMat.SubMat(flowerWithoutGlyph); var whiteOcrResult = OcrFactory.Paddle.OcrResult(roi); flowerName = whiteOcrResult.Text; // 所以只好识别两次，Trim后根据字数取原截图OCR的结果…… @@ -242,7 +244,7 @@ public class GetGridIconsTask : ISoloTask double width = 60; double height = 60; // 宽高缩放似乎不一致，似乎在2.05:2.15之间，但不知道怎么测定 Rect iconRect = new Rect((int)(itemRegion.Width / 2 - 237 * scale - width / 2), (int)(itemRegion.Height / 2 - height / 2), (int)width, (int)height); - Mat crop = itemRegion.SrcMat.SubMat(iconRect); + using Mat crop = itemRegion.SrcMat.SubMat(iconRect); return crop.Resize(new Size(125, 125)); } diff --git a/BetterGenshinImpact/GameTask/GetGridIcons/GridIconsAccuracyTestTask.cs b/BetterGenshinImpact/GameTask/GetGridIcons/GridIconsAccuracyTestTask.cs index 7cfefb15..08e20ac6 100644 --- a/BetterGenshinImpact/GameTask/GetGridIcons/GridIconsAccuracyTestTask.cs +++ b/BetterGenshinImpact/GameTask/GetGridIcons/GridIconsAccuracyTestTask.cs @@ -113,8 +113,9 @@ public class GridIconsAccuracyTestTask : ISoloTask gridScreen.OnBeforeScroll += () => VisionContext.Instance().DrawContent.ClearAll(); try { - await foreach (ImageRegion itemRegion in gridScreen) + await foreach ((ImageRegion pageRegion, Rect itemRect) in gridScreen) { + using ImageRegion itemRegion = pageRegion.DeriveCrop(itemRect); itemRegion.Click(); Task task1 = Delay(300, ct); diff --git a/BetterGenshinImpact/GameTask/Model/GameUI/ArtifactSetFilterScreen.cs b/BetterGenshinImpact/GameTask/Model/GameUI/ArtifactSetFilterScreen.cs index 5430bae0..9945a342 100644 --- a/BetterGenshinImpact/GameTask/Model/GameUI/ArtifactSetFilterScreen.cs +++ b/BetterGenshinImpact/GameTask/Model/GameUI/ArtifactSetFilterScreen.cs @@ -10,10 +10,11 @@ using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; +using TorchSharp.Modules; namespace BetterGenshinImpact.GameTask.Model.GameUI { - public class ArtifactSetFilterScreen : IAsyncEnumerable + public class ArtifactSetFilterScreen : IAsyncEnumerable> { private readonly GridParams @params; private readonly CancellationToken ct; @@ -36,22 +37,22 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI this.logger = logger; this.@params = @params; } - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + public IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) { return new GridEnumerator(this, @params.Roi, @params.Columns, new GridScroller(@params, logger, input, ct), ct); } - public class GridEnumerator : IAsyncEnumerator + public class GridEnumerator : IAsyncEnumerator> { private readonly ArtifactSetFilterScreen owner; private readonly Rect roi; private readonly CancellationToken ct; private readonly int columns; private readonly GridScroller gridScroller; - - private Queue imageRegions; - private ImageRegion? current; - ImageRegion IAsyncEnumerator.Current => current ?? throw new NullReferenceException(); + private record Page(ImageRegion PageRegion, Queue ItemRects); + private Page? currentPage; + private Tuple? current; + Tuple IAsyncEnumerator>.Current => current ?? throw new NullReferenceException(); internal GridEnumerator(ArtifactSetFilterScreen owner, Rect roi, int columns, GridScroller gridScroller, CancellationToken ct) { @@ -60,15 +61,13 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI this.ct = ct; this.columns = columns; this.gridScroller = gridScroller; - - this.imageRegions = new Queue(); } public async ValueTask MoveNextAsync() { - if (current == null || this.imageRegions.Count < 1) + if (this.currentPage == null || this.currentPage.ItemRects.Count < 1) { - if (current != null) + if (this.currentPage != null) { using var ra4 = TaskControl.CaptureToRectArea(); ra4.MoveTo(this.roi.X + this.roi.Width / 2, this.roi.Y + this.roi.Height / 2); @@ -82,17 +81,34 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI } using ImageRegion ra = TaskControl.CaptureToRectArea(); - using ImageRegion imageRegion = ra.DeriveCrop(this.roi); - IEnumerable gridItems = GetGridItems(imageRegion.SrcMat, this.columns).Select(imageRegion.DeriveCrop); - this.imageRegions = new Queue(gridItems); + ImageRegion imageRegion = ra.DeriveCrop(this.roi); + try + { + IEnumerable gridRects = GetGridItems(imageRegion.SrcMat, this.columns); + + if (!gridRects.Any()) + { + imageRegion.Dispose(); + return false; + } + + this.currentPage?.PageRegion?.Dispose(); + this.currentPage = new Page(imageRegion, new Queue(gridRects)); + } + catch + { + imageRegion?.Dispose(); + throw; + } } - this.current = this.imageRegions.Dequeue(); + this.current = Tuple.Create(this.currentPage.PageRegion, this.currentPage.ItemRects.Dequeue()); return true; } public ValueTask DisposeAsync() { + this.currentPage?.PageRegion?.Dispose(); return ValueTask.CompletedTask; } } @@ -134,131 +150,11 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI IEnumerable boxes = contours.Select(Cv2.BoundingRect); - double avgWidth = boxes.Average(r => r.Width); - double avgHeight = boxes.Average(r => r.Height); + List cells = GridCell.ClusterToCells(boxes, 10).ToList(); - List cells = ClusterToCells(boxes, 10).ToList(); - - double avgColSpacing; - double avgRowSpace; - { - int count = 0; - int sum = 0; - foreach (var row in cells.GroupBy(t => t.RowNum)) - { - for (int i = 0; i < row.Max(r => r.ColNum); i++) - { - var x1 = row.SingleOrDefault(r => r.ColNum == i); - var x2 = row.SingleOrDefault(r => r.ColNum == i + 1); - if (x1 == null || x2 == null) - { - continue; - } - sum += x2.Rect.X - x1.Rect.X - x1.Rect.Width; - count++; - } - } - avgColSpacing = Math.Round(((double)sum) / count, MidpointRounding.AwayFromZero); - } - { - int count = 0; - int sum = 0; - foreach (var col in cells.GroupBy(t => t.ColNum)) - { - for (int i = 0; i < col.Max(r => r.RowNum); i++) - { - var y1 = col.SingleOrDefault(r => r.RowNum == i); - var y2 = col.SingleOrDefault(r => r.RowNum == i + 1); - if (y1 == null || y2 == null) - { - continue; - } - sum += y2.Rect.Y - y1.Rect.Y - y1.Rect.Height; - count++; - } - } - avgRowSpace = Math.Round(((double)sum) / count, MidpointRounding.AwayFromZero); - } - - int avgLeft = (int)Math.Round(cells.Average(c => c.Rect.X - (avgWidth + avgColSpacing) * c.ColNum), MidpointRounding.AwayFromZero); - int avgTop = (int)Math.Round(cells.Average(c => c.Rect.Y - (avgHeight + avgRowSpace) * c.RowNum), MidpointRounding.AwayFromZero); - - // 遍历方阵，补上缺的Cell - for (int i = 0; i < cells.Max(r => r.ColNum) + 1; i++) - { - for (int j = 0; j < cells.Max(r => r.RowNum) + 1; j++) - { - if (cells.Any(c => c.ColNum == i && c.RowNum == j)) - { - continue; - } - int x = (int)Math.Round(avgLeft + (avgWidth + avgColSpacing) * i, MidpointRounding.AwayFromZero); - int y = (int)Math.Round(avgTop + (avgHeight + avgRowSpace) * j, MidpointRounding.AwayFromZero); - int width = (int)Math.Round(avgWidth, MidpointRounding.AwayFromZero); - int height = (int)Math.Round(avgHeight, MidpointRounding.AwayFromZero); - Cell cell = new Cell(new Rect(x, y, width, height)); - cell.ColNum = i; - cell.RowNum = j; - cells.Add(cell); - } - } + GridCell.FillMissingGridCells(ref cells); return cells.OrderBy(c => c.RowNum).ThenBy(c => c.ColNum).Select(c => c.Rect).ToArray(); } - - /// - /// 具有行号列号的单元格 - /// ColNum和RowNum也是0-based的 - /// 不仅方便编程，ClusterToCells方法也需要一个引用类型 - /// - /// - private class Cell(Rect rect) - { - internal Rect Rect = rect; - internal int ColNum; - internal int RowNum; - } - - /// - /// 把检出的矩形聚簇成类似Excel的单元格集合 - /// - /// - /// - /// - private static IEnumerable ClusterToCells(IEnumerable rects, int threshold) - { - var result = rects.Select(r => new Cell(r)); - result = result.ToArray(); // 必需，不然引用会丢掉。。 - - var orderByX = result.OrderBy(t => t.Rect.Left).ToArray(); - int col = 0; - int? lastX = null; - int avgWidth = (int)rects.Average(r => r.Width); - for (int i = 0; i < orderByX.Length; i++) - { - if (lastX != null && orderByX[i].Rect.X - lastX > threshold) - { - col += (int)Math.Round((float)(orderByX[i].Rect.X - lastX.Value) / (avgWidth + threshold)); - } - orderByX[i].ColNum = col; - lastX = orderByX[i].Rect.X; - } - - var orderByY = result.OrderBy(t => t.Rect.Top).ToArray(); - int row = 0; - int? lastY = null; - int avgHeight = (int)rects.Average(r => r.Height); - for (int i = 0; i < orderByY.Length; i++) - { - if (lastY != null && orderByY[i].Rect.Y - lastY > threshold) - { - row += (int)Math.Round((float)(orderByY[i].Rect.Y - lastY.Value) / (avgHeight + threshold)); // 估算隔了多少行 - } - orderByY[i].RowNum = row; - lastY = orderByY[i].Rect.Y; - } - - return result; - } } } diff --git a/BetterGenshinImpact/GameTask/Model/GameUI/GridCell.cs b/BetterGenshinImpact/GameTask/Model/GameUI/GridCell.cs new file mode 100644 index 00000000..149e39b9 --- /dev/null +++ b/BetterGenshinImpact/GameTask/Model/GameUI/GridCell.cs @@ -0,0 +1,162 @@ +using OpenCvSharp; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace BetterGenshinImpact.GameTask.Model.GameUI +{ + /// + /// 具有行号列号的单元格 + /// ColNum和RowNum也是0-based的 + /// 不仅方便编程，ClusterToCells方法也需要一个引用类型 + /// + /// + public class GridCell(Rect rect) + { + public Rect Rect = rect; + public int ColNum; + public int RowNum; + /// + /// 表示该单元格并非CV方法识别得到，而是通过算法推测出的 + /// + public bool IsPhantom; + + /// + /// 把检出的矩形聚簇成类似Excel的单元格集合 + /// + /// + /// + /// + public static IEnumerable ClusterToCells(IEnumerable rects, int threshold) + { + return ClusterToCells(rects.Select(r => Tuple.Create(0, r)), threshold).Select(t => t.Item2).ToArray(); + } + + /// + /// 把检出的矩形聚簇成类似Excel的单元格集合 + /// + /// + /// + /// + public static IEnumerable> ClusterToCells(IEnumerable> rects, int threshold) + { + if (!rects.Any()) + { + return []; + } + + var result = rects.Select(r => new Tuple(r.Item1, new GridCell(r.Item2))); + result = result.ToArray(); // 必需，不然引用会丢掉。。 + + var orderByX = result.OrderBy(t => t.Item2.Rect.Left).ToArray(); + int col = 0; + int? lastX = null; + int avgWidth = (int)rects.Average(r => r.Item2.Width); + for (int i = 0; i < orderByX.Length; i++) + { + if (lastX != null && orderByX[i].Item2.Rect.X - lastX > threshold) + { + col += (int)Math.Round((float)(orderByX[i].Item2.Rect.X - lastX.Value) / (avgWidth + threshold)); + } + orderByX[i].Item2.ColNum = col; + lastX = orderByX[i].Item2.Rect.X; + } + + var orderByY = result.OrderBy(t => t.Item2.Rect.Top).ToArray(); + int row = 0; + int? lastY = null; + int avgHeight = (int)rects.Average(r => r.Item2.Height); + for (int i = 0; i < orderByY.Length; i++) + { + if (lastY != null && orderByY[i].Item2.Rect.Y - lastY > threshold) + { + row += (int)Math.Round((float)(orderByY[i].Item2.Rect.Y - lastY.Value) / (avgHeight + threshold)); // 估算隔了多少行 + } + orderByY[i].Item2.RowNum = row; + lastY = orderByY[i].Item2.Rect.Y; + } + + return result; + } + + /// + /// 遍历方阵，补上缺的Cell + /// + /// + public static void FillMissingGridCells(ref List cells) + { + if (cells.Count <= 0) + { + return; + } + + double avgWidth = cells.Average(c => c.Rect.Width); + double avgHeight = cells.Average(c => c.Rect.Height); + double avgColSpacing; + double avgRowSpace; + { + int count = 0; + int sum = 0; + foreach (var row in cells.GroupBy(t => t.RowNum)) + { + for (int i = 0; i < row.Max(r => r.ColNum); i++) + { + var x1 = row.SingleOrDefault(r => r.ColNum == i); + var x2 = row.SingleOrDefault(r => r.ColNum == i + 1); + if (x1 == null || x2 == null) + { + continue; + } + sum += x2.Rect.X - x1.Rect.X - x1.Rect.Width; + count++; + } + } + avgColSpacing = count == 0 ? 0 : Math.Round(((double)sum) / count, MidpointRounding.AwayFromZero); + } + { + int count = 0; + int sum = 0; + foreach (var col in cells.GroupBy(t => t.ColNum)) + { + for (int i = 0; i < col.Max(r => r.RowNum); i++) + { + var y1 = col.SingleOrDefault(r => r.RowNum == i); + var y2 = col.SingleOrDefault(r => r.RowNum == i + 1); + if (y1 == null || y2 == null) + { + continue; + } + sum += y2.Rect.Y - y1.Rect.Y - y1.Rect.Height; + count++; + } + } + avgRowSpace = count == 0 ? 0 : Math.Round(((double)sum) / count, MidpointRounding.AwayFromZero); + } + + int avgLeft = (int)Math.Round(cells.Average(c => c.Rect.X - (avgWidth + avgColSpacing) * c.ColNum), MidpointRounding.AwayFromZero); + int avgTop = (int)Math.Round(cells.Average(c => c.Rect.Y - (avgHeight + avgRowSpace) * c.RowNum), MidpointRounding.AwayFromZero); + + for (int i = 0; i < cells.Max(r => r.ColNum) + 1; i++) + { + for (int j = 0; j < cells.Max(r => r.RowNum) + 1; j++) + { + if (cells.Any(c => c.ColNum == i && c.RowNum == j)) + { + continue; + } + int x = (int)Math.Round(avgLeft + (avgWidth + avgColSpacing) * i, MidpointRounding.AwayFromZero); + int y = (int)Math.Round(avgTop + (avgHeight + avgRowSpace) * j, MidpointRounding.AwayFromZero); + int width = (int)Math.Round(avgWidth, MidpointRounding.AwayFromZero); + int height = (int)Math.Round(avgHeight, MidpointRounding.AwayFromZero); + GridCell cell = new GridCell(new Rect(x, y, width, height)) + { + ColNum = i, + RowNum = j, + IsPhantom = true + }; + cells.Add(cell); + } + } + } + } +} diff --git a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs index 7220d7fd..abd74a22 100644 --- a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs +++ b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs @@ -14,23 +14,25 @@ using System.Threading.Tasks; namespace BetterGenshinImpact.GameTask.Model.GameUI { - public class GridScreen : IAsyncEnumerable + public class GridScreen : IAsyncEnumerable> { private readonly GridParams @params; private readonly CancellationToken ct; private readonly ILogger logger; private readonly InputSimulator input = Simulation.SendInput; internal Action? OnBeforeScroll { get; set; } - internal Action>? OnAfterTurnToNewPage { get; set; } + internal Action>>>? OnAfterTurnToNewPage { get; set; } /// /// 提供一个默认的绘制页面上所有识别出的项目的行为 /// - internal static readonly Action> DrawItemsAfterTurnToNewPage = items => + internal static readonly Action>>> DrawItemsAfterTurnToNewPage = data => { - foreach (ImageRegion item in items) + (ImageRegion page, var items) = data; + foreach ((Rect rect, bool isPhantom) in items) { - item.DrawSelf($"GridItem{item.GetHashCode()}", System.Drawing.Pens.Lime); + using ImageRegion item = page.DeriveCrop(rect); + item.DrawSelf($"GridItem{item.GetHashCode()}", isPhantom ? System.Drawing.Pens.Yellow : System.Drawing.Pens.Lime); } }; @@ -47,19 +49,19 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI { this.ct = ct; this.logger = logger; - if (@params.Columns < 4) + if (@params.Columns < 3) { throw new ArgumentOutOfRangeException(nameof(@params.Columns)); } this.@params = @params; } - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + public IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) { return new GridEnumerator(this, @params.Roi, @params.Columns, input, new GridScroller(@params, logger, input, ct), ct); } - public class GridEnumerator : IAsyncEnumerator + public class GridEnumerator : IAsyncEnumerator> { private readonly GridScreen owner; private readonly Rect roi; @@ -73,10 +75,10 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI /// /// 供枚举输出的队列 /// 为了防止Grid的页面元素自动回收复用技术导致item高亮干扰，每次滚动后记录靠近下方的一个item，在下次滚动前主动点击该item - private record Page(Queue ImageRegions, Rect? AntiRecycling); + private record Page(ImageRegion PageRegion, Queue ItemRects, Rect? AntiRecycling); private Page? currentPage; - private ImageRegion? current; - ImageRegion IAsyncEnumerator.Current => current ?? throw new NullReferenceException(); + private Tuple? current; + Tuple IAsyncEnumerator>.Current => current ?? throw new NullReferenceException(); /// /// 滚动操作枚举器 @@ -101,86 +103,7 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI } /// - /// 将图标按Y轴高度简单地进行聚簇，避免因微小差异而乱序 - /// 已知每行的图标之间的Y不会差得太多 - /// - /// 传入的Y列表 - /// - /// 外层是各行从上到下，内层是一行从左到右 - public static List> ClusterRows(IEnumerable regions, int threshold) - { - static int getX(T t) - { - if (t is ImageRegion imageRegion) - { - return imageRegion.X; - } - else if (t is Rect rect) - { - return rect.X; - } - else - { - throw new NotSupportedException(); - } - } - static int getY(T t) - { - if (t is ImageRegion imageRegion) - { - return imageRegion.Y; - } - else if (t is Rect rect) - { - return rect.Y; - } - else - { - throw new NotSupportedException(); - } - } - - // 先对Y排序，便于聚簇 - var sortedRegions = regions.OrderBy(getY).ToList(); - - List> clusters = new List>(); - - if (sortedRegions.Count == 0) - return clusters; - - // 初始化第一个聚簇 - List currentCluster = new List { }; - - foreach (T r in sortedRegions) - { - if (currentCluster.Count <= 0) - { - currentCluster.Add(r); - continue; - } - - T lastInCluster = currentCluster.Last(); - - // 如果当前数字与聚簇中最后一个数字的差值小于阈值，则加入当前聚簇 - if (getY(r) - getY(lastInCluster) <= threshold) - { - currentCluster.Add(r); - } - else - { - // 否则，创建一个新的聚簇 - clusters.Add(currentCluster.OrderBy(getX).ToList()); - currentCluster = new List { r }; - } - } - - // 添加最后一个聚簇 - clusters.Add(currentCluster.OrderBy(getX).ToList()); - - return clusters; - } - - /// + /// 纯cv方法获取 /// 返回未经排序的所有GridItem /// /// @@ -230,7 +153,7 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI { return false; } - return Math.Abs((float)r.Width / r.Height - 0.81) < 0.05; // 按形状筛选 + return Math.Abs((float)r.Width / r.Height - 0.81) < 0.03; // 按形状筛选 }).ToArray(); IEnumerable boxes = contours.Select(Cv2.BoundingRect); @@ -395,80 +318,149 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI return contours; } + /// + /// 把Rects结果聚簇成Cells，并进行优化 + /// + /// + /// + /// + /// + public static IEnumerable PostProcess(Mat mat, IEnumerable rects, int threshold) + { + if (!rects.Any()) + { + return []; + } + // 根据聚簇结果补漏…… + List cells = GridCell.ClusterToCells(rects, threshold).ToList(); + GridCell.FillMissingGridCells(ref cells); + + // 在末尾处有可能补多了，把底部颜色不符的丢掉…… // PS：群友有直接用底部颜色进行识别的，效果不错 + var result = cells.ToList(); + foreach (var cell in cells.Where(c => c.IsPhantom)) + { + using Mat cellMat = mat.SubMat(cell.Rect); + using Mat bottom = cellMat.GetGridBottom(); + if (!IsCorrectBottomColor(bottom)) + { + result.Remove(cell); + } + } + + return result; + } + public async ValueTask MoveNextAsync() { - if (this.currentPage == null || this.currentPage.ImageRegions.Count < 1) + if (this.currentPage == null || this.currentPage.ItemRects.Count < 1) { - IEnumerable gridItems; - if (this.currentPage != null) // 当前页遍历完了就向下滚动 + ImageRegion? imageRegion = null; + try { - if (this.currentPage.AntiRecycling.HasValue) + if (this.currentPage != null) // 当前页遍历完了就向下滚动 { + if (this.currentPage.AntiRecycling.HasValue) + { + using DesktopRegion desktop = new DesktopRegion(this.input.Mouse); + var (x, y, w, h) = (this.currentPage.AntiRecycling.Value.X, this.currentPage.AntiRecycling.Value.Y, this.currentPage.AntiRecycling.Value.Width, this.currentPage.AntiRecycling.Value.Height); + var (gcX, gcY) = (TaskContext.Instance().SystemInfo.CaptureAreaRect.X, TaskContext.Instance().SystemInfo.CaptureAreaRect.Y); + desktop.ClickTo(gcX + this.roi.X + x + (w / 2d), gcY + this.roi.Y + y + (h / 2d)); + await TaskControl.Delay(500, ct); + desktop.ClickTo(gcX + this.roi.X + x + (w / 2d), gcY + this.roi.Y + y + (h / 2d)); + await TaskControl.Delay(500, ct); + } + + using var ra4 = TaskControl.CaptureToRectArea(); + ra4.MoveTo(this.roi.X + this.roi.Width / 2, this.roi.Y + this.roi.Height / 2); + await TaskControl.Delay(300, ct); + + owner.OnBeforeScroll?.Invoke(); + if (!await this.gridScroller.TryVerticalScollDown((src, columns) => GetGridItems(src, columns))) + { + return false; + } + + using ImageRegion ra = TaskControl.CaptureToRectArea(); + imageRegion = ra.DeriveCrop(this.roi); + } + else + { + // 第一页采集时，主动操作来避免图标高亮 + Rect rect12 = new Rect(0, 0, (int)(this.roi.Width * 1.5 / this.columns), this.roi.Height); + // 双击第三列，采集第一、二列 using DesktopRegion desktop = new DesktopRegion(this.input.Mouse); - var (x, y, w, h) = (this.currentPage.AntiRecycling.Value.X, this.currentPage.AntiRecycling.Value.Y, this.currentPage.AntiRecycling.Value.Width, this.currentPage.AntiRecycling.Value.Height); var (gcX, gcY) = (TaskContext.Instance().SystemInfo.CaptureAreaRect.X, TaskContext.Instance().SystemInfo.CaptureAreaRect.Y); - desktop.ClickTo(gcX + this.roi.X + x + (w / 2d), gcY + this.roi.Y + y + (h / 2d)); + desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 2.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); + await TaskControl.Delay(300, ct); + desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 2.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); await TaskControl.Delay(500, ct); - desktop.ClickTo(gcX + this.roi.X + x + (w / 2d), gcY + this.roi.Y + y + (h / 2d)); + + using ImageRegion ra12 = TaskControl.CaptureToRectArea(); + using ImageRegion imageRegion12 = ra12.DeriveCrop(this.roi); + using Mat columns12 = new Mat(imageRegion12.SrcMat, rect12); + + // 双击第一列，采集第二列以后的列 + desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 0.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); + await TaskControl.Delay(300, ct); + desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 0.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); await TaskControl.Delay(500, ct); + + using ImageRegion raRest = TaskControl.CaptureToRectArea(); + imageRegion = raRest.DeriveCrop(this.roi); + using Mat subMat12 = imageRegion.SrcMat.SubMat(rect12); + columns12.CopyTo(subMat12); // 拼接两次的采集 } - using var ra4 = TaskControl.CaptureToRectArea(); - ra4.MoveTo(this.roi.X + this.roi.Width / 2, this.roi.Y + this.roi.Height / 2); - await TaskControl.Delay(300, ct); + var rects = GetGridItems(imageRegion.SrcMat, this.columns); + var cells = PostProcess(imageRegion.SrcMat, rects, (int)(0.025 * this.roi.Height)); - owner.OnBeforeScroll?.Invoke(); - if (!await this.gridScroller.TryVerticalScollDown((src, columns) => GetGridItems(src, columns))) + if (!cells.Any()) { + imageRegion.Dispose(); return false; } - using ImageRegion ra = TaskControl.CaptureToRectArea(); - using ImageRegion imageRegion = ra.DeriveCrop(this.roi); - gridItems = GetGridItems(imageRegion.SrcMat, this.columns).Select(imageRegion.DeriveCrop); + this.currentPage?.PageRegion?.Dispose(); + this.currentPage = new Page(imageRegion, new Queue(cells.OrderBy(c => c.RowNum).ThenBy(c => c.ColNum).Select(c => c.Rect)), + cells.GroupBy(c => c.RowNum).OrderByDescending(g => g.Key).Skip(1)?.FirstOrDefault()?.OrderBy(c => c.ColNum)?.FirstOrDefault()?.Rect); + + owner.OnAfterTurnToNewPage?.Invoke(Tuple.Create(imageRegion, cells.Select(c => Tuple.Create(c.Rect, c.IsPhantom)))); } - else + catch { - // 第一页采集时，主动操作来避免图标高亮 - // 双击第四列，采集第一、二列 - using DesktopRegion desktop = new DesktopRegion(this.input.Mouse); - var (gcX, gcY) = (TaskContext.Instance().SystemInfo.CaptureAreaRect.X, TaskContext.Instance().SystemInfo.CaptureAreaRect.Y); - desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 3.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); - await TaskControl.Delay(500, ct); - desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 3.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); - await TaskControl.Delay(500, ct); - - using ImageRegion ra12 = TaskControl.CaptureToRectArea(); - using ImageRegion imageRegion12 = ra12.DeriveCrop(this.roi); - using Mat columns12 = new Mat(imageRegion12.SrcMat, new Rect(0, 0, (int)(this.roi.Width * 2.5 / this.columns), this.roi.Height)); - IEnumerable columns12Items = GetGridItems(columns12, 2); - // 双击第一列，采集第三列以后的列 - desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 0.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); - await TaskControl.Delay(500, ct); - desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 0.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); - await TaskControl.Delay(500, ct); - - using ImageRegion raRest = TaskControl.CaptureToRectArea(); - using ImageRegion imageRegionRest = raRest.DeriveCrop(this.roi); - int restStartX = (int)(this.roi.Width * 1.5 / this.columns); - using Mat columnsRest = new Mat(imageRegionRest.SrcMat, new Rect(restStartX, 0, this.roi.Width - restStartX, this.roi.Height)); - IEnumerable columnsRestItems = GetGridItems(columnsRest, this.columns - 2).Select(r => new Rect(r.X + restStartX, r.Y, r.Width, r.Height)); - - gridItems = columns12Items.Select(imageRegion12.DeriveCrop).Union(columnsRestItems.Select(imageRegionRest.DeriveCrop)).ToArray(); + imageRegion?.Dispose(); + throw; } - - List> clusterRows = ClusterRows(gridItems, (int)(0.025 * this.roi.Height)); - this.currentPage = new Page(new Queue(clusterRows.SelectMany(r => r)), clusterRows.Reverse>().Skip(1)?.FirstOrDefault()?.FirstOrDefault()?.ToRect()); - - owner.OnAfterTurnToNewPage?.Invoke(this.currentPage.ImageRegions); } - this.current = this.currentPage.ImageRegions.Dequeue(); + this.current = Tuple.Create(this.currentPage.PageRegion, this.currentPage.ItemRects.Dequeue()); return true; } + /// + /// 使用均值比较颜色 + /// + public static bool IsCorrectBottomColor(Mat image, int tolerance = 30) + { + if (image.Empty()) + throw new ArgumentException("输入图像为空"); + + Scalar bgrColor = new Scalar(0xdc, 0xe5, 0xe9); + + // 计算区域的平均颜色 + Scalar meanColor = Cv2.Mean(image); + + // 计算平均颜色与目标颜色的差异 + double diff = Math.Abs(meanColor.Val0 - bgrColor.Val0) + + Math.Abs(meanColor.Val1 - bgrColor.Val1) + + Math.Abs(meanColor.Val2 - bgrColor.Val2); + + return diff <= tolerance * 3; + } + public ValueTask DisposeAsync() { + this.currentPage?.PageRegion?.Dispose(); return ValueTask.CompletedTask; } } diff --git a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenExtensions.cs b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenExtensions.cs index e2a1c32b..a0991d98 100644 --- a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenExtensions.cs +++ b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenExtensions.cs @@ -15,7 +15,7 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI /// public static string GetGridItemIconText(this Mat mat, IOcrService ocrService) { - Mat subMat = mat.SubMat(mat.Height * 128 / 153, mat.Height * 150 / 153, mat.Width * 5 / 125, mat.Width * 120 / 125); + using Mat subMat = mat.SubMat(mat.Height * 128 / 153, mat.Height * 150 / 153, mat.Width * 5 / 125, mat.Width * 120 / 125); using Mat resize = subMat.Resize(new Size(subMat.Width * 2, subMat.Height * 2)); return ocrService.Ocr(resize); } @@ -28,7 +28,18 @@ namespace BetterGenshinImpact.GameTask.Model.GameUI public static Mat GetGridIcon(this Mat mat) { using Mat resized = mat.Resize(new Size(125, 153)); - return resized.SubMat(0, 125, 0, 125).Clone(); + return resized.SubMat(0, 125, 0, 125); + } + + /// + /// 截取Grid图标中底部的部分 + /// + /// + /// + public static Mat GetGridBottom(this Mat mat) + { + using Mat resized = mat.Resize(new Size(125, 153)); + return resized.SubMat(126, 153, 0, 125); } } } diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs index 1259b259..5b9ff091 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs @@ -173,12 +173,18 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoArtifactSalvageTests new ArtifactAffix(ArtifactAffixType.ElementalMastery, 23), new ArtifactAffix(ArtifactAffixType.ATKPercent, 4.1f) ], 0), new CultureInfo("zh-Hant") }; - yield return new object[] { "20250828093344_GetArtifactStat.png", new ArtifactStat("黃金時代的先聲",new ArtifactAffix(ArtifactAffixType.DEFPercent, 8.7f), [ + yield return new object[] { "20250828093344_GetArtifactStat.png", new ArtifactStat("黃金時代的先聲", new ArtifactAffix(ArtifactAffixType.DEFPercent, 8.7f), [ new ArtifactAffix(ArtifactAffixType.DEF, 19), new ArtifactAffix(ArtifactAffixType.CRITDMG, 7.8f), new ArtifactAffix(ArtifactAffixType.ATK, 18), new ArtifactAffix(ArtifactAffixType.ElementalMastery, 23) ], 0), new CultureInfo("zh-Hant") }; + yield return new object[] { "202510311559_GetArtifactStat.png", new ArtifactStat("黃金乐曲的变奏",/* 应为"黄"*/ new ArtifactAffix(ArtifactAffixType.HP, 717f), [ + new ArtifactAffix(ArtifactAffixType.DEF, 16), + new ArtifactAffix(ArtifactAffixType.ElementalMastery, 16), + new ArtifactAffix(ArtifactAffixType.ATKPercent, 4.1f), + new ArtifactAffix(ArtifactAffixType.HPPercent, 4.7f) + ], 0), new CultureInfo("zh-Hans") }; } } @@ -261,7 +267,7 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoArtifactSalvageTests timeProvider.Advance(TimeSpan.FromSeconds(3)); // - await Assert.ThrowsAsync(()=> sut); + await Assert.ThrowsAsync(() => sut); } } } diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/GetGridIconsTests/GridIconsAccuracyTestTaskTests.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/GetGridIconsTests/GridIconsAccuracyTestTaskTests.cs index 4d84245b..1f2dc175 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/GetGridIconsTests/GridIconsAccuracyTestTaskTests.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/GetGridIconsTests/GridIconsAccuracyTestTaskTests.cs @@ -43,19 +43,16 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.GetGridIconsTests using Mat mat = new Mat(@$"..\..\..\Assets\{screenshot}"); var gridItems = GridScreen.GridEnumerator.GetGridItems(mat, columns, findContoursAlpha: findContoursAlpha); - var rows = GridScreen.GridEnumerator.ClusterRows(gridItems, 10); + var cells = GridCell.ClusterToCells(gridItems, 10).OrderBy(c => c.RowNum).ThenBy(c => c.ColNum); // var result = new List<(string?, int)>(); - foreach (var row in rows) + foreach (var cell in cells) { - foreach (Rect rect in row) - { - Mat gridItemMat = mat.SubMat(rect); - using Mat icon = gridItemMat.GetGridIcon(); - var pred = GridIconsAccuracyTestTask.Infer(icon, this.session, this.prototypes); - result.Add(pred); - } + using Mat gridItemMat = mat.SubMat(cell.Rect); + using Mat icon = gridItemMat.GetGridIcon(); + var pred = GridIconsAccuracyTestTask.Infer(icon, this.session, this.prototypes); + result.Add(pred); } // diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs index 23cb235d..43128902 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs @@ -20,9 +20,11 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.Model.GameUI [Theory] [InlineData(@"AutoArtifactSalvage\ArtifactGrid.png", 4, 2)] + [InlineData(@"GetGridIcons\FoodGrid.png", 32, 8)] + [InlineData(@"GetGridIcons\WeaponGrid.png", 4, 2)] [InlineData(@"GetGridIcons\WeaponGrid3.png", 32, 8)] /// - /// 测试获取各种界面中的物品图标，结果应正确 + /// 测试获取各种界面中的物品图标，经过算法的后处理，结果应正确 /// public void GetGridIcons_ShouldBeRight(string screenshot, int count, int columns) { @@ -30,18 +32,20 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.Model.GameUI using Mat mat = new Mat(@$"..\..\..\Assets\{screenshot}"); // - var result = GridScreen.GridEnumerator.GetGridItems(mat, columns); + var rects = GridScreen.GridEnumerator.GetGridItems(mat, columns); + var cells = GridScreen.GridEnumerator.PostProcess(mat, rects, (int)(0.025 * mat.Height)); // - Assert.Equal(count, result.Count()); + Assert.Equal(count, cells.Count()); } [Theory] + [InlineData(@"AutoArtifactSalvage\ArtifactGrid.png", 4, 2)] [InlineData(@"GetGridIcons\FoodGrid.png", 32, 8)] [InlineData(@"GetGridIcons\WeaponGrid.png", 4, 2)] [InlineData(@"GetGridIcons\WeaponGrid3.png", 32, 8)] /// - /// 测试获取各种界面中的物品图标，使用复杂的cv算法，结果应正确 + /// 测试获取各种界面中的物品图标，使用复杂的cv算法，经过算法的后处理，结果应正确 /// public void GetGridIconsAlpha_ShouldBeRight(string screenshot, int count, int columns) { @@ -49,10 +53,11 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.Model.GameUI using Mat mat = new Mat(@$"..\..\..\Assets\{screenshot}"); // - var result = GridScreen.GridEnumerator.GetGridItems(mat, columns, findContoursAlpha: true); + var rects = GridScreen.GridEnumerator.GetGridItems(mat, columns, findContoursAlpha: true); + var cells = GridScreen.GridEnumerator.PostProcess(mat, rects, (int)(0.025 * mat.Height)); // - Assert.Equal(count, result.Count()); + Assert.Equal(count, cells.Count()); } [Fact] @@ -101,17 +106,14 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.Model.GameUI // var gridItems = GridScreen.GridEnumerator.GetGridItems(mat, columns, findContoursAlpha: true); - var rows = GridScreen.GridEnumerator.ClusterRows(gridItems, 10); + var cells = GridCell.ClusterToCells(gridItems, 10).OrderBy(c => c.RowNum).ThenBy(c => c.ColNum); var result = new List(); - foreach (var row in rows) + foreach (var cell in cells) { - foreach (Rect rect in row) - { - Mat gridItemMat = mat.SubMat(rect); - string numStr = gridItemMat.GetGridItemIconText(paddle.Get()); - result.Add(numStr); - } + using Mat gridItemMat = mat.SubMat(cell.Rect); + string numStr = gridItemMat.GetGridItemIconText(paddle.Get()); + result.Add(numStr); } //