diff --git a/BetterGenshinImpact/Core/Config/AllConfig.cs b/BetterGenshinImpact/Core/Config/AllConfig.cs index bc9dbef7..d1624e3b 100644 --- a/BetterGenshinImpact/Core/Config/AllConfig.cs +++ b/BetterGenshinImpact/Core/Config/AllConfig.cs @@ -20,6 +20,7 @@ using System.Threading.Tasks; using BetterGenshinImpact.GameTask.AutoTrackPath; using BetterGenshinImpact.GameTask.AutoArtifactSalvage; using BetterGenshinImpact.GameTask.AutoStygianOnslaught; +using BetterGenshinImpact.GameTask.GetGridIcons; namespace BetterGenshinImpact.Core.Config; @@ -164,6 +165,11 @@ public partial class AllConfig : ObservableObject /// public AutoArtifactSalvageConfig AutoArtifactSalvageConfig { get; set; } = new(); + /// + /// 截取物品图标配置 + /// + public GetGridIconsConfig GetGridIconsConfig { get; set; } = new(); + /// /// 宏配置 /// diff --git a/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs b/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs index 638d8cf9..52410031 100644 --- a/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs +++ b/BetterGenshinImpact/GameTask/AutoArtifactSalvage/AutoArtifactSalvageTask.cs @@ -24,7 +24,7 @@ using BetterGenshinImpact.Core.Recognition.OpenCv; using BetterGenshinImpact.Core.Recognition.OCR; using BetterGenshinImpact.GameTask.Common; using BetterGenshinImpact.GameTask.Common.Job; -using GameTask.Model.GameUI; +using BetterGenshinImpact.GameTask.Model.GameUI; namespace BetterGenshinImpact.GameTask.AutoArtifactSalvage; @@ -227,8 +227,9 @@ public class AutoArtifactSalvageTask : ISoloTask int count = maxNumToCheck; using var ra0 = CaptureToRectArea(); - Rect gridRoi = new Rect((int)(ra0.Width * 0.025), (int)(ra0.Width * 0.055), (int)(ra0.Width * 0.66), (int)(ra0.Width * 0.4)); - GridScreen gridScreen = new GridScreen(gridRoi, 3, 40, 28, 0.018, this.logger, this.ct); // 圣遗物分解Grid有4行9列 + GridScreenParams gridParams = GridScreenParams.Templates[GridScreenName.ArtifactSalvage]; + Rect gridRoi = gridParams.GetRect(ra0); + GridScreen gridScreen = new GridScreen(gridRoi, gridParams, this.logger, this.ct); // 圣遗物分解Grid有4行9列 await foreach (ImageRegion itemRegion in gridScreen) { Rect gridRect = itemRegion.ToRect(); diff --git a/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsConfig.cs b/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsConfig.cs new file mode 100644 index 00000000..feadb5a2 --- /dev/null +++ b/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsConfig.cs @@ -0,0 +1,28 @@ +using BetterGenshinImpact.GameTask.Model.GameUI; +using CommunityToolkit.Mvvm.ComponentModel; +using System; + +namespace BetterGenshinImpact.GameTask.GetGridIcons; + +[Serializable] +public partial class GetGridIconsConfig : ObservableObject +{ + /// + /// 昼夜策略 + /// 钓全天的鱼、还是只钓白天或夜晚的鱼 + /// + [ObservableProperty] + private GridScreenName _gridName = GridScreenName.Weapons; + + // 使用星星作为后缀 + [ObservableProperty] + private bool _starAsSuffix = false; + + // 使用等级作为后缀 + [ObservableProperty] + private bool _lvAsSuffix = false; + + // 最多获取多少个图标 + [ObservableProperty] + private int _maxNumToGet = int.MaxValue; +} \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsTask.cs b/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsTask.cs new file mode 100644 index 00000000..c4451cc5 --- /dev/null +++ b/BetterGenshinImpact/GameTask/GetGridIcons/GetGridIconsTask.cs @@ -0,0 +1,107 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using BetterGenshinImpact.Core.Simulator; +using Microsoft.Extensions.Logging; +using static BetterGenshinImpact.GameTask.Common.TaskControl; +using Microsoft.Extensions.Localization; +using System.Globalization; +using BetterGenshinImpact.GameTask.Model.Area; +using System.Collections.Generic; +using Fischless.WindowsInput; +using OpenCvSharp; +using System.Linq; +using BetterGenshinImpact.GameTask.Common.Job; +using BetterGenshinImpact.Core.Recognition.OCR; +using System.IO; +using System.Drawing; +using static Vanara.PInvoke.Gdi32; +using OpenCvSharp.Extensions; +using BetterGenshinImpact.GameTask.Model.GameUI; + +namespace BetterGenshinImpact.GameTask.GetGridIcons; + +/// +/// 获取Grid界面的物品图标 +/// +public class GetGridIconsTask : ISoloTask +{ + private readonly ILogger logger = App.GetLogger(); + private readonly InputSimulator input = Simulation.SendInput; + private readonly ReturnMainUiTask _returnMainUiTask = new(); + + private CancellationToken ct; + + public string Name => "获取Grid界面物品图标独立任务"; + + private readonly int? maxNumToGet; + + private readonly GridScreenName gridScreenName; + + public GetGridIconsTask(GridScreenName gridScreenName, int? maxNumToGet = null) + { + this.gridScreenName = gridScreenName; + this.maxNumToGet = maxNumToGet; + IStringLocalizer stringLocalizer = App.GetService>() ?? throw new NullReferenceException(); + CultureInfo cultureInfo = new CultureInfo(TaskContext.Instance().Config.OtherConfig.GameCultureInfoName); + } + + public async Task Start(CancellationToken ct) + { + this.ct = ct; + + using var ra0 = CaptureToRectArea(); + GridScreenParams gridParams = GridScreenParams.Templates[this.gridScreenName]; + Rect gridRoi = gridParams.GetRect(ra0); + + int count = this.maxNumToGet ?? int.MaxValue; + + string directory = Path.Combine(AppContext.BaseDirectory, "log/gridIcons", DateTime.Now.ToString("yyyyMMddHHmmss")); + Directory.CreateDirectory(directory); + + GridScreen gridScreen = new GridScreen(gridRoi, gridParams, this.logger, this.ct); + HashSet itemNames = new HashSet(); + await foreach (ImageRegion itemRegion in gridScreen) + { + itemRegion.Click(); + await Delay(300, ct); + + using var ra1 = CaptureToRectArea(); + using ImageRegion nameRegion = ra1.DeriveCrop(new Rect((int)(ra1.Width * 0.682), (int)(ra1.Width * 0.0625), (int)(ra1.Width * 0.256), (int)(ra1.Width * 0.03125))); + var ocrResult = OcrFactory.Paddle.OcrResult(nameRegion.SrcMat); + string itemName = ocrResult.Text; + if (itemNames.Add(itemName)) + { + string filePath = Path.Combine(directory, $"{itemName}.png"); + Thread saveThread = new Thread(() => + { + try + { + using (FileStream fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) + { + itemRegion.SrcMat.ToBitmap().Save(fs, System.Drawing.Imaging.ImageFormat.Png); + } + logger.LogInformation("图片保存成功:{Text}", itemName); + } + catch (Exception e) + { + logger.LogError(e, "图片保存失败:{Text}", itemName); + } + }); + saveThread.IsBackground = true; // 设置为后台线程 + saveThread.Start(); + } + else + { + logger.LogInformation("重复的物品:{Text}", itemName); + } + + count--; + if (count <= 0) + { + logger.LogInformation("检查次数已耗尽"); + break; + } + } + } +} \ No newline at end of file diff --git a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs index 5b24d52f..82f6e74a 100644 --- a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs +++ b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreen.cs @@ -1,4 +1,5 @@ using BetterGenshinImpact.Core.Simulator; +using BetterGenshinImpact.GameTask; using BetterGenshinImpact.GameTask.Common; using BetterGenshinImpact.GameTask.Model.Area; using Fischless.WindowsInput; @@ -12,7 +13,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -namespace GameTask.Model.GameUI +namespace BetterGenshinImpact.GameTask.Model.GameUI { public class GridScreen : IAsyncEnumerable { @@ -20,6 +21,7 @@ namespace GameTask.Model.GameUI private readonly CancellationToken ct; private readonly ILogger logger; private readonly InputSimulator input = Simulation.SendInput; + private readonly int columns; private readonly int s1Round; private readonly int roundMilliseconds; private readonly int s2Round; @@ -34,20 +36,25 @@ namespace GameTask.Model.GameUI /// Grid所在位置 /// /// - public GridScreen(Rect gridRoi, int s1Round, int roundMilliseconds, int s2Round, double s3Scale, ILogger logger, CancellationToken ct) + public GridScreen(Rect gridRoi, GridScreenParams @params, ILogger logger, CancellationToken ct) { this.gridRoi = gridRoi; this.ct = ct; this.logger = logger; - this.s1Round = s1Round; - this.roundMilliseconds = roundMilliseconds; - this.s2Round = s2Round; - this.s3Scale = s3Scale; + if (@params.Columns < 4) + { + throw new ArgumentOutOfRangeException(nameof(@params.Columns)); + } + this.columns = @params.Columns; + this.s1Round = @params.S1Round; + this.roundMilliseconds = @params.RoundMilliseconds; + this.s2Round = @params.S2Round; + this.s3Scale = @params.S3Scale; } public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { - return new GridEnumerator(this.gridRoi, this.s1Round, this.roundMilliseconds, this.s2Round, this.s3Scale, this.logger, this.input, this.ct); + return new GridEnumerator(gridRoi, columns, s1Round, roundMilliseconds, s2Round, s3Scale, logger, input, ct); } public class GridEnumerator : IAsyncEnumerator @@ -56,12 +63,18 @@ namespace GameTask.Model.GameUI private readonly CancellationToken ct; private readonly ILogger logger; private readonly InputSimulator input = Simulation.SendInput; + private readonly int columns; private readonly int s1Round; private readonly int roundMilliseconds; private readonly int s2Round; private readonly double s3Scale; - private record Page(ImageRegion ImageRegion, Stack Rects); + /// + /// 单次滚动得到的页面 + /// + /// 供枚举输出的队列 + /// 为了防止Grid的页面元素自动回收复用技术导致item高亮干扰,每次滚动后记录靠近下方的一个item,在下次滚动前主动点击该item + private record Page(Queue ImageRegions, Rect? AntiRecycling); private Page? currentPage; private ImageRegion current; ImageRegion IAsyncEnumerator.Current => current; @@ -70,6 +83,7 @@ namespace GameTask.Model.GameUI /// 滚动操作枚举器 /// /// + /// 有几列 /// 测试是否能滚动时发出的滚动命令次数 /// 滚动命令间隔毫秒 /// 滚过一整页时发出的滚动命令次数 @@ -77,12 +91,13 @@ namespace GameTask.Model.GameUI /// /// /// - public GridEnumerator(Rect roi, int s1Round, int roundMilliseconds, int s2Round, double s3Scale, ILogger logger, InputSimulator input, CancellationToken ct) + public GridEnumerator(Rect roi, int columns, int s1Round, int roundMilliseconds, int s2Round, double s3Scale, ILogger logger, InputSimulator input, CancellationToken ct) { this.roi = roi; this.ct = ct; this.logger = logger; this.input = input; + this.columns = columns; this.s1Round = s1Round; this.roundMilliseconds = roundMilliseconds; this.s2Round = s2Round; @@ -115,7 +130,7 @@ namespace GameTask.Model.GameUI /// 尝试滚动并等待可能的回弹后的灰度图 /// 估计的位移 /// 低于下限则可能不存在平移 - /// 上限用于抵消微小的其他差异 + /// 上限用于抵消微小的其他差异,比如高亮图标的呼吸闪烁 /// /// public static bool IsScrolling(Mat prevGray, Mat nextGray, out Point2d shift, double lowerThreshold = 0.5, double upperThreshold = 0.95, ILogger? logger = null) @@ -127,42 +142,284 @@ namespace GameTask.Model.GameUI using Mat window = new Mat(); shift = Cv2.PhaseCorrelate(prev, next, window, out double response); // 相位相关性 - logger?.LogInformation($"response={response:F3}, shift=({shift.X:F2}, {shift.Y:F2})"); + //logger?.LogInformation($"response={response:F3}, shift=({shift.X:F2}, {shift.Y:F2})"); return response > lowerThreshold && response < upperThreshold; } - public static IEnumerable GetGridItems(Mat src) + /// + /// 将图标按Y轴高度简单地进行聚簇,避免因微小差异而乱序 + /// 已知每行的图标之间的Y不会差得太多 + /// + /// 传入的Y列表 + /// + /// 外层是各行从上到下,内层是一行从左到右 + static List> ClusterRows(IEnumerable regions, int threshold) { - using Mat grey = src.CvtColor(ColorConversionCodes.BGR2GRAY); + // 先对Y排序,便于聚簇 + var sortedRegions = regions.OrderBy(r => r.Y).ToList(); - using Mat canny = grey.Canny(20, 40); + List> clusters = new List>(); - Cv2.FindContours(canny, out var contours, out _, RetrievalModes.External, ContourApproximationModes.ApproxSimple, null); + if (sortedRegions.Count == 0) + return clusters; - IEnumerable boxes = contours.Where(c => Cv2.MinAreaRect(c).Angle % 90 <= 1) // 剔除倾斜 - .Select(Cv2.BoundingRect).Where(r => + // 初始化第一个聚簇 + List currentCluster = new List { }; + + foreach (ImageRegion r in sortedRegions) + { + if (currentCluster.Count <= 0) { + currentCluster.Add(r); + continue; + } + + ImageRegion lastInCluster = currentCluster.Last(); + + // 如果当前数字与聚簇中最后一个数字的差值小于阈值,则加入当前聚簇 + if (r.Y - lastInCluster.Y <= threshold) + { + currentCluster.Add(r); + } + else + { + // 否则,创建一个新的聚簇 + clusters.Add(currentCluster.OrderBy(r => r.X).ToList()); + currentCluster = new List { r }; + } + } + + // 添加最后一个聚簇 + clusters.Add(currentCluster.OrderBy(r => r.X).ToList()); + + return clusters; + } + + /// + /// 返回未经排序的所有GridItem + /// + /// + /// + /// + /// + public static IEnumerable GetGridItems(Mat src, int columns, bool findContoursAlpha = false) + { + Point[][] contours = findContoursAlpha ? FindContoursAlpha(src) : FindContours(src); + + IEnumerable Crop() + { + foreach (var contour in contours) + { + Rect rect = Cv2.BoundingRect(contour); + + // 把右上角的点去掉 + var topRightPoints = contour.Where(p => (p.X - rect.X) > (rect.Width * 0.60) && (p.Y - rect.Y) < (rect.Height * 0.28)); + + yield return contour.Except(topRightPoints).ToArray(); + } + } + + contours = Crop().ToArray(); + + //foreach (var c in contours) + //{ + // RotatedRect rect = Cv2.MinAreaRect(c); + // Point2f[] rectPoints = rect.Points(); + // Point[] rectPointsInt = Array.ConvertAll(rectPoints, p => new Point((int)p.X, (int)p.Y)); + // // 在图像上绘制最小外接旋转矩形 + // for (int i = 0; i < 4; i++) + // { + // Cv2.Line(src, rectPointsInt[i], rectPointsInt[(i + 1) % 4], Scalar.Pink, 2); + // } + //} + + contours = contours + .Where(c => + { + Rect r = Cv2.BoundingRect(c); + if (r.Width < src.Width / columns * 0.66) // 剔除太小的 + { + return false; + } if (r.Height == 0) { return false; } return Math.Abs((float)r.Width / r.Height - 0.8) < 0.05; // 按形状筛选 - }).ToList(); + }).ToArray(); - //src.DrawContours(contours, -1, Scalar.Red); + IEnumerable boxes = contours.Select(Cv2.BoundingRect); - int biggestRectHeight = boxes.Max(b => b.Height); - boxes = boxes.Where(b => (float)b.Height / biggestRectHeight > 0.88); // 剔除太小的 + //if (boxes.Count() != 32) + //{ + // src.DrawContours(contours, -1, Scalar.Red); + // foreach (Rect box in boxes) + // { + // Cv2.Rectangle(src, box.TopLeft, box.BottomRight, Scalar.AliceBlue); + // } + // Cv2.ImShow("src", src); + // Cv2.WaitKey(); + // Cv2.DestroyAllWindows(); + //} - return boxes.ToArray(); + return boxes; + } + + /// + /// 像“分解圣遗物”界面的背景是纯色的,用简单的算法就能提取轮廓 + /// + /// + /// + public static Point[][] FindContours(Mat src) + { + using Mat grey = src.CvtColor(ColorConversionCodes.BGR2GRAY); + //Cv2.ImShow("grey", grey); + + using Mat canny = grey.Canny(20, 40); + //Cv2.ImShow("canny", canny); + //Cv2.WaitKey(); + + //闭运算把一些断裂的边缘粘合一下 + //局限:提纳里的耳朵太长了一直连到了正上方的另一个图标,这里闭运算就会把最后一丝空隙也连起来,仅凭亮度边缘无法分隔轮廓…… + //todo:使用头像识别,先行去掉头像 + using Mat closeKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(5, 5)); + using Mat close = canny.MorphologyEx(MorphTypes.Close, closeKernel); + //Cv2.ImShow("close", close); + //Cv2.WaitKey(); + + Cv2.FindContours(close, out Point[][] contours, out _, RetrievalModes.External, ContourApproximationModes.ApproxNone, null); + return contours; + } + + /// + /// 背包界面的背景是把打开界面之前的画面进行了模糊+黑白渐变滤镜+左上角水印叠加处理 + /// 放任五彩斑斓的输入,并且允许点击高亮的话处理起来就复杂了 + /// 所以这个Alpha版方法留在这里只是想说明: + /// 越是琢磨算法,就越会发现传统算法的能力是有极限的 + /// 既然是游戏画面,不如在输入的时候就尽量获得没有噪声的画面 + /// + /// + /// + public static Point[][] FindContoursAlpha(Mat src) + { + Point[][] contours; + void getLine(Mat edge, Scalar color) + { + using Mat threshold = edge.Threshold(30, 255, ThresholdTypes.Binary); + LineSegmentPoint[] lines = threshold.HoughLinesP(1, (Cv2.PI / 180) / 4, 100, maxLineGap: 3); + lines = lines.Where(l => (Math.Abs(l.P1.X - l.P2.X) == 0) || (Math.Abs(l.P1.Y - l.P2.Y) == 0)).ToArray(); + foreach (var line in lines) + { + Cv2.Line(src, line.P1, line.P2, color, 1); + } + } + + Mat Laplacian(Mat src) + { + //拉普拉斯算子 + //Canny的sobel算子太偏向于横平竖直,而在噪声干扰太厉害的地方会产生分叉 + using Mat laplacian = src.Laplacian(MatType.CV_64F, ksize: 3); + Mat result = laplacian.ConvertScaleAbs(); + return result; + } + + using Mat edge = new Mat(src.Size(), MatType.CV_8UC1); + edge.SetTo(0); + + //除了明度,饱和度也纳入考虑 + //另外色度带来的噪声实在太多了所以不用,但其实有些地方色度的边缘比另两个维度的边缘好得多 + using Mat hsv = src.CvtColor(ColorConversionCodes.BGR2HSV); + + using Mat satChannel = hsv.ExtractChannel(1); + //Cv2.ImShow("satChannel", satChannel); + using Mat satEdge = Laplacian(satChannel); + Cv2.BitwiseOr(satEdge, edge, edge); + //Cv2.ImShow("satEdge", satEdge); + //getLine(satEdge, Scalar.Red); + + using Mat valChannel = hsv.ExtractChannel(2); + //Cv2.ImShow("valChannel", valChannel); + using Mat valEdge = Laplacian(valChannel); + Cv2.BitwiseOr(valEdge, edge, edge); + //Cv2.ImShow("valEdge", valEdge); + //getLine(valEdge, Scalar.Lime); + + //Cv2.WaitKey(); + //Cv2.ImShow("edge", edge); + //Cv2.WaitKey(); + + //高斯模糊方便去噪点 + //但毕竟是模糊,轮廓会被扩大,并且很难说是均匀的扩大 + using Mat blurred = edge.GaussianBlur(new Size(3, 3), 0.5); + //Cv2.ImShow("blurred", blurred); + //Cv2.WaitKey(); + + //合并明度饱和度的边缘后再二值化 + Mat threshold = blurred.Threshold(50, 255, ThresholdTypes.Binary); + //Cv2.ImShow("threshold", threshold); + //Cv2.WaitKey(); + + /* + * 如果不用高斯模糊去噪点,自己搞一些形态学操作也行 + * 有些边缘会比高斯效果好 + //把太小的轮廓丢掉 + //Cv2.FindContours(threshold, out contours, out _, RetrievalModes.External, ContourApproximationModes.ApproxSimple, null); + contours = contours.Where(c => + { + Rect rect = Cv2.BoundingRect(c); + if ((rect.Width > 10) || (rect.Height > 10)) + { + return true; + } + return false; + }).ToArray(); + threshold.SetTo(0); + threshold.DrawContours(contours, -1, Scalar.White, thickness: 1); + Cv2.ImShow("threshold", threshold); + Cv2.WaitKey(); + + //闭运算把一些断裂的边缘粘合一下 + using Mat closeKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(5, 5)); + using Mat close = threshold.MorphologyEx(MorphTypes.Close, closeKernel); + Cv2.ImShow("close", close); + Cv2.WaitKey(); + + //因为后面要做的开运算会把毛刺给去掉,但太细的边缘会被一起腐蚀掉,所以查找并填充一下轮廓 + Cv2.FindContours(close, out contours, out _, RetrievalModes.External, ContourApproximationModes.ApproxSimple, null); + close.DrawContours(contours, -1, Scalar.White, thickness: -1); + Cv2.ImShow("close", close); + Cv2.WaitKey(); + + //开运算去毛刺 + using Mat openKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(5, 5)); + using Mat open = close.MorphologyEx(MorphTypes.Open, openKernel); + Cv2.ImShow("open", open); + Cv2.WaitKey(); + */ + + //得到有噪点的边缘 + Cv2.FindContours(threshold, out contours, out _, RetrievalModes.External, ContourApproximationModes.ApproxNone, null); + + return contours; } public async ValueTask MoveNextAsync() { - if (this.currentPage == null || this.currentPage.Rects.Count < 1) + if (this.currentPage == null || this.currentPage.ImageRegions.Count < 1) { if (this.currentPage != null) { + if (this.currentPage.AntiRecycling.HasValue) + { + using DesktopRegion desktop = new DesktopRegion(this.input.Mouse); + var (x, y, w, h) = (this.currentPage.AntiRecycling.Value.X, this.currentPage.AntiRecycling.Value.Y, this.currentPage.AntiRecycling.Value.Width, this.currentPage.AntiRecycling.Value.Height); + var (gcX, gcY) = (TaskContext.Instance().SystemInfo.CaptureAreaRect.X, TaskContext.Instance().SystemInfo.CaptureAreaRect.Y); + desktop.ClickTo(gcX + this.roi.X + x + (w / 2d), gcY + this.roi.Y + y + (h / 2d)); + await TaskControl.Delay(500, ct); + desktop.ClickTo(gcX + this.roi.X + x + (w / 2d), gcY + this.roi.Y + y + (h / 2d)); + await TaskControl.Delay(500, ct); + } + //BetterGenshinImpact.View.Drawable.VisionContext.Instance().DrawContent.ClearAll(); using var ra4 = TaskControl.CaptureToRectArea(); @@ -185,7 +442,7 @@ namespace GameTask.Model.GameUI await TaskControl.Delay(60, ct); using var ra2 = TaskControl.CaptureToRectArea(); using ImageRegion grid2 = ra2.DeriveCrop(this.roi); - IEnumerable gridItems2 = GetGridItems(grid2.SrcMat); + IEnumerable gridItems2 = GetGridItems(grid2.SrcMat, this.columns); if (gridItems2.Min(i => i.Y) > (ra2.Width * this.s3Scale)) // 最后精细滚动,保证完整地显示最多行 { input.Mouse.VerticalScroll(-1); @@ -208,18 +465,53 @@ namespace GameTask.Model.GameUI } } - using var ra = TaskControl.CaptureToRectArea(); - var imageRegion = ra.DeriveCrop(this.roi); - IEnumerable gridItems = GetGridItems(imageRegion.SrcMat); - this.currentPage = new Page(imageRegion, new Stack(gridItems)); + IEnumerable gridItems; + if (this.currentPage == null) + { + // 第一页采集时,主动操作来避免图标高亮 + // 双击第四列,采集第一、二列 + using DesktopRegion desktop = new DesktopRegion(this.input.Mouse); + var (gcX, gcY) = (TaskContext.Instance().SystemInfo.CaptureAreaRect.X, TaskContext.Instance().SystemInfo.CaptureAreaRect.Y); + desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 3.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); + await TaskControl.Delay(500, ct); + desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 3.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); + await TaskControl.Delay(500, ct); - //foreach (Rect item in gridItems) + using ImageRegion ra12 = TaskControl.CaptureToRectArea(); + using ImageRegion imageRegion12 = ra12.DeriveCrop(this.roi); + using Mat columns12 = new Mat(imageRegion12.SrcMat, new Rect(0, 0, (int)(this.roi.Width * 2.5 / this.columns), this.roi.Height)); + IEnumerable columns12Items = GetGridItems(columns12, 2); + // 双击第一列,采集第三列以后的列 + desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 0.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); + await TaskControl.Delay(500, ct); + desktop.ClickTo(gcX + this.roi.X + this.roi.Width * 0.5 / this.columns, gcY + this.roi.Y + this.roi.Width * 0.5 / this.columns); + await TaskControl.Delay(500, ct); + + using ImageRegion raRest = TaskControl.CaptureToRectArea(); + using ImageRegion imageRegionRest = raRest.DeriveCrop(this.roi); + int restStartX = (int)(this.roi.Width * 1.5 / this.columns); + using Mat columnsRest = new Mat(imageRegionRest.SrcMat, new Rect(restStartX, 0, this.roi.Width - restStartX, this.roi.Height)); + IEnumerable columnsRestItems = GetGridItems(columnsRest, this.columns - 2).Select(r => new Rect(r.X + restStartX, r.Y, r.Width, r.Height)); + + gridItems = columns12Items.Select(imageRegion12.DeriveCrop).Union(columnsRestItems.Select(imageRegionRest.DeriveCrop)).ToArray(); + } + else + { + using ImageRegion ra = TaskControl.CaptureToRectArea(); + using ImageRegion imageRegion = ra.DeriveCrop(this.roi); + gridItems = GetGridItems(imageRegion.SrcMat, this.columns).Select(imageRegion.DeriveCrop); + } + + List> clusterRows = ClusterRows(gridItems, (int)(0.025 * this.roi.Height)); + this.currentPage = new Page(new Queue(clusterRows.SelectMany(r => r)), clusterRows.Reverse>().Skip(1)?.FirstOrDefault()?.FirstOrDefault()?.ToRect()); + + //foreach (Rect item in gridItems.Select(r => r.ToRect())) //{ - // imageRegion.DrawRect(item, item.GetHashCode().ToString(), new System.Drawing.Pen(System.Drawing.Color.Blue)); + // imageRegion.DrawRect(item, item.GetHashCode().ToString(), new System.Drawing.Pen(System.Drawing.Color.Lime)); //} } - this.current = this.currentPage.ImageRegion.DeriveCrop(this.currentPage.Rects.Pop()); + this.current = this.currentPage.ImageRegions.Dequeue(); return true; } diff --git a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenName.cs b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenName.cs new file mode 100644 index 00000000..c7ba0d4e --- /dev/null +++ b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenName.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text; + +namespace BetterGenshinImpact.GameTask.Model.GameUI +{ + public enum GridScreenName + { + [Description("武器")] + Weapons, + [Description("圣遗物")] + Artifacts, + [Description("养成道具")] + CharacterDevelopmentItems, + [Description("食物")] + Food, + [Description("材料")] + Materials, + [Description("小道具")] + Gadget, + [Description("任务")] + Quest, + [Description("贵重道具")] + PreciousItems, + [Description("摆设")] + Furnishings, + [Description("圣遗物分解")] + ArtifactSalvage + } +} diff --git a/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenParams.cs b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenParams.cs new file mode 100644 index 00000000..eaca7376 --- /dev/null +++ b/BetterGenshinImpact/GameTask/Model/GameUI/GridScreenParams.cs @@ -0,0 +1,74 @@ +using BetterGenshinImpact.GameTask.Model.Area; +using OpenCvSharp; +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Text; + +namespace BetterGenshinImpact.GameTask.Model.GameUI +{ + public class GridScreenParams + { + internal int X1080p { get; private set; } + internal int Y1080p { get; private set; } + internal int Width1080p { get; private set; } + internal int Height1080p { get; private set; } + internal int Columns { get; private set; } + internal int S1Round { get; private set; } + internal int RoundMilliseconds { get; private set; } + internal int S2Round { get; private set; } + internal double S3Scale { get; private set; } + + internal Rect GetRect(ImageRegion gameScreen) + { + float scale = gameScreen.Height / 1080f; + return new Rect((int)(scale * X1080p), (int)(scale * Y1080p), (int)(scale * Width1080p), (int)(scale * Height1080p)); + } + + private static readonly GridScreenParams weapons = new GridScreenParams() + { + X1080p = 106, + Y1080p = 110, + Width1080p = 1171, + Height1080p = 845, + Columns = 8, + S1Round = 3, + RoundMilliseconds = 40, + S2Round = 32, + S3Scale = 0.024 + }; + + internal static FrozenDictionary Templates { get; } = new Dictionary() { + { GridScreenName.Weapons, weapons }, + { GridScreenName.Artifacts, new GridScreenParams(){ + X1080p = 106, + Y1080p = 162, + Width1080p = 1171, + Height1080p = 783, + Columns = 8, + S1Round = 3, + RoundMilliseconds = 40, + S2Round = 32, + S3Scale = 0.024 + }}, + { GridScreenName.CharacterDevelopmentItems, weapons }, + { GridScreenName.Food, weapons }, + { GridScreenName.Materials, weapons }, + { GridScreenName.Gadget, weapons }, + { GridScreenName.Quest, weapons }, + { GridScreenName.PreciousItems, weapons }, + { GridScreenName.Furnishings, weapons }, + { GridScreenName.ArtifactSalvage, new GridScreenParams(){ + X1080p = 48, + Y1080p = 106, + Width1080p = 1267, + Height1080p = 768, + Columns = 9, + S1Round = 3, + RoundMilliseconds = 40, + S2Round = 28, + S3Scale = 0.018 + }} + }.ToFrozenDictionary(); + } +} diff --git a/BetterGenshinImpact/View/Pages/TaskSettingsPage.xaml b/BetterGenshinImpact/View/Pages/TaskSettingsPage.xaml index 41c4deeb..c869a701 100644 --- a/BetterGenshinImpact/View/Pages/TaskSettingsPage.xaml +++ b/BetterGenshinImpact/View/Pages/TaskSettingsPage.xaml @@ -75,7 +75,7 @@ Grid.Column="0" Foreground="{ui:ThemeResource TextFillColorTertiaryBrush}" TextWrapping="Wrap"> - 全自动打牌 - + 全自动打牌 - 点击查看使用教程 @@ -1251,10 +1251,10 @@ Grid.Column="1" MinWidth="90" Margin="0,0,36,0" /> - + - + @@ -1346,7 +1346,7 @@ - + @@ -1502,7 +1502,7 @@ IsChecked="{Binding Config.AutoStygianOnslaughtConfig.AutoArtifactSalvage, Mode=TwoWay}" /> - + @@ -1809,8 +1809,8 @@ Maximum="60" Minimum="5" ValidationMode="InvalidInputOverwritten" - Value="{Binding Config.AutoFishingConfig.AutoThrowRodTimeOut, Mode=TwoWay}" - Text="{Binding Config.AutoFishingConfig.AutoThrowRodTimeOut, Mode=TwoWay}" /> + Value="{Binding Config.AutoFishingConfig.AutoThrowRodTimeOut, Mode=TwoWay, UpdateSourceTrigger=PropertyChanged}" + Text="{Binding Config.AutoFishingConfig.AutoThrowRodTimeOut, Mode=TwoWay, UpdateSourceTrigger=PropertyChanged}" /> @@ -1838,8 +1838,8 @@ Maximum="1800" Minimum="0" ValidationMode="InvalidInputOverwritten" - Value="{Binding Config.AutoFishingConfig.WholeProcessTimeoutSeconds, Mode=TwoWay}" - Text="{Binding Config.AutoFishingConfig.WholeProcessTimeoutSeconds, Mode=TwoWay}" /> + Value="{Binding Config.AutoFishingConfig.WholeProcessTimeoutSeconds, Mode=TwoWay, UpdateSourceTrigger=PropertyChanged}" + Text="{Binding Config.AutoFishingConfig.WholeProcessTimeoutSeconds, Mode=TwoWay, UpdateSourceTrigger=PropertyChanged}" /> @@ -1919,7 +1919,7 @@ Foreground="{ui:ThemeResource TextFillColorSecondaryBrush}"> 下载 到本地后填入torch_cpu.dll或torch_cuda.dll的完整地址。如未生效可尝试重启BGI。 - + + Value="{Binding Config.AutoArtifactSalvageConfig.MaxNumToCheck, Mode=TwoWay, UpdateSourceTrigger=PropertyChanged}" + Text="{Binding Config.AutoArtifactSalvageConfig.MaxNumToCheck, Mode=TwoWay, UpdateSourceTrigger=PropertyChanged}" /> + + + + + + + + + + + + + + + + + + + + + + + 须要打开设置-启用保存截图功能,文件保存在 + + log/gridIcons + + + 以下过长的内容在pr时会搬到教程里去 + + 需要漆黑的背景以降低干扰,比如渊下宫-蛇肠之路的一个锚点,将视角竖直向上看向洞顶 + + 诸如提纳里的耳朵太长了,他装备的物品角标目前无法正确地和正上方的物品图标进行轮廓分割,请手动规避 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/BetterGenshinImpact/ViewModel/Pages/TaskSettingsPageViewModel.cs b/BetterGenshinImpact/ViewModel/Pages/TaskSettingsPageViewModel.cs index cf41070a..8df36076 100644 --- a/BetterGenshinImpact/ViewModel/Pages/TaskSettingsPageViewModel.cs +++ b/BetterGenshinImpact/ViewModel/Pages/TaskSettingsPageViewModel.cs @@ -35,6 +35,8 @@ using System.Diagnostics; using BetterGenshinImpact.GameTask.AutoArtifactSalvage; using BetterGenshinImpact.GameTask.AutoStygianOnslaught; using BetterGenshinImpact.View.Windows; +using BetterGenshinImpact.GameTask.GetGridIcons; +using BetterGenshinImpact.GameTask.Model.GameUI; namespace BetterGenshinImpact.ViewModel.Pages; @@ -80,7 +82,7 @@ public partial class TaskSettingsPageViewModel : ViewModel [ObservableProperty] private string _switchAutoDomainButtonText = "启动"; - + [ObservableProperty] private int _autoStygianOnslaughtRoundNum; @@ -124,7 +126,7 @@ public partial class TaskSettingsPageViewModel : ViewModel [ObservableProperty] private AutoFightViewModel? _autoFightViewModel; - + [ObservableProperty] private OneDragonFlowViewModel? _oneDragonFlowViewModel; @@ -154,6 +156,20 @@ public partial class TaskSettingsPageViewModel : ViewModel [ObservableProperty] private bool _switchArtifactSalvageEnabled; + [ObservableProperty] + private bool _switchGetGridIconsEnabled; + [ObservableProperty] + private string _switchGetGridIconsButtonText = "启动"; + [ObservableProperty] + private FrozenDictionary _gridNameDict = Enum.GetValues(typeof(GridScreenName)) + .Cast() + .ToFrozenDictionary( + e => (Enum)e, + e => e.GetType() + .GetField(e.ToString())? + .GetCustomAttribute()? + .Description ?? e.ToString()); + public TaskSettingsPageViewModel(IConfigService configService, INavigationService navigationService, TaskTriggerDispatcher taskTriggerDispatcher) { Config = configService.Get(); @@ -168,21 +184,21 @@ public partial class TaskSettingsPageViewModel : ViewModel _autoFightViewModel = new AutoFightViewModel(Config); _oneDragonFlowViewModel = new OneDragonFlowViewModel(); } - - + + [RelayCommand] private async Task OnSOneDragonFlow() - { - if (OneDragonFlowViewModel == null || OneDragonFlowViewModel.SelectedConfig == null) - { + { + if (OneDragonFlowViewModel == null || OneDragonFlowViewModel.SelectedConfig == null) + { OneDragonFlowViewModel.OnNavigatedTo(); if (OneDragonFlowViewModel == null || OneDragonFlowViewModel.SelectedConfig == null) { Toast.Warning("未设置任务!"); return; } - } - await OneDragonFlowViewModel.OnOneKeyExecute(); + } + await OneDragonFlowViewModel.OnOneKeyExecute(); } [RelayCommand] @@ -349,7 +365,7 @@ public partial class TaskSettingsPageViewModel : ViewModel .RunSoloTaskAsync(new AutoStygianOnslaughtTask(Config.AutoStygianOnslaughtConfig, path)); SwitchAutoStygianOnslaughtEnabled = false; } - + [RelayCommand] private async Task OnGoToAutoStygianOnslaughtUrlAsync() { @@ -505,4 +521,30 @@ public partial class TaskSettingsPageViewModel : ViewModel OcrDialog ocrDialog = new OcrDialog(0.70, 0.098, 0.24, 0.52, "圣遗物分解", this.Config.AutoArtifactSalvageConfig.RegularExpression); ocrDialog.ShowDialog(); } + + [RelayCommand] + private async Task OnSwitchGetGridIcons() + { + try + { + SwitchGetGridIconsEnabled = true; + await new TaskRunner().RunSoloTaskAsync(new GetGridIconsTask(Config.GetGridIconsConfig.GridName, Config.GetGridIconsConfig.MaxNumToGet)); + } + finally + { + SwitchGetGridIconsEnabled = false; + } + } + + [RelayCommand] + private void OnGoToGridIconsFolder() + { + var path = Global.Absolute(@"log\gridIcons\"); + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + } + + Process.Start("explorer.exe", path); + } } \ No newline at end of file diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs index 7f01e39e..3ccd7ffb 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/AutoArtifactSalvageTests/AutoArtifactSalvageTaskTests.cs @@ -1,6 +1,6 @@ using BetterGenshinImpact.GameTask.AutoArtifactSalvage; +using BetterGenshinImpact.GameTask.Model.GameUI; using BetterGenshinImpact.UnitTest.CoreTests.RecognitionTests.OCRTests; -using GameTask.Model.GameUI; using OpenCvSharp; using System; using System.Collections.Concurrent; @@ -42,24 +42,6 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoArtifactSalvageTests Assert.Equal(status, result); } - [Theory] - [InlineData(@"ArtifactGrid.png")] - /// - /// 测试获取分解圣遗物Grid界面中的圣遗物,结果应正确 - /// - public void GetArtifactGridItems_ShouldBeRight(string screenshot) - { - // - using Mat mat = new Mat(@$"..\..\..\Assets\AutoArtifactSalvage\{screenshot}"); - - // - var result = GridScreen.GridEnumerator.GetGridItems(mat); - - // - Assert.Equal(4, result.Count()); - } - - [Fact] /// /// 测试获取分解圣遗物Grid界面中的圣遗物,以及其状态,结果应正确 @@ -70,7 +52,7 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.AutoArtifactSalvageTests using Mat mat = new Mat(@$"..\..\..\Assets\AutoArtifactSalvage\ArtifactGrid.png"); // - var result = GridScreen.GridEnumerator.GetGridItems(mat); + var result = GridScreen.GridEnumerator.GetGridItems(mat, 2); using Mat leftTopOne = new Mat(mat, result.Single(r => r.X + r.Width / 2 < mat.Width / 2 && r.Y + r.Height / 2 < mat.Height / 2)); using Mat rightTopOne = new Mat(mat, result.Single(r => r.X + r.Width / 2 > mat.Width / 2 && r.Y + r.Height / 2 < mat.Height / 2)); using Mat leftBottomOne = new Mat(mat, result.Single(r => r.X + r.Width / 2 < mat.Width / 2 && r.Y + r.Height / 2 > mat.Height / 2)); diff --git a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs index 0d599679..1f9ed583 100644 --- a/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs +++ b/Test/BetterGenshinImpact.UnitTest/GameTaskTests/Model/GameUI/GridScreenTests.cs @@ -1,4 +1,4 @@ -using GameTask.Model.GameUI; +using BetterGenshinImpact.GameTask.Model.GameUI; using OpenCvSharp; using System; using System.Collections.Generic; @@ -10,6 +10,43 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.Model.GameUI { public class GridScreenTests { + [Theory] + [InlineData(@"AutoArtifactSalvage\ArtifactGrid.png", 4, 2)] + [InlineData(@"GetGridIcons\WeaponGrid3.png", 32, 8)] + /// + /// 测试获取各种界面中的物品图标,结果应正确 + /// + public void GetGridIcons_ShouldBeRight(string screenshot, int count, int columns) + { + // + using Mat mat = new Mat(@$"..\..\..\Assets\{screenshot}"); + + // + var result = GridScreen.GridEnumerator.GetGridItems(mat, columns); + + // + Assert.Equal(count, result.Count()); + } + + [Theory] + [InlineData(@"GetGridIcons\FoodGrid.png", 32, 8)] + [InlineData(@"GetGridIcons\WeaponGrid.png", 4, 2)] + [InlineData(@"GetGridIcons\WeaponGrid3.png", 32, 8)] + /// + /// 测试获取各种界面中的物品图标,使用复杂的cv算法,结果应正确 + /// + public void GetGridIconsAlpha_ShouldBeRight(string screenshot, int count, int columns) + { + // + using Mat mat = new Mat(@$"..\..\..\Assets\{screenshot}"); + + // + var result = GridScreen.GridEnumerator.GetGridItems(mat, columns, findContoursAlpha: true); + + // + Assert.Equal(count, result.Count()); + } + [Fact] /// /// 测试判断前后两张图是否属于滚动,结果应正确 @@ -25,7 +62,7 @@ namespace BetterGenshinImpact.UnitTest.GameTaskTests.Model.GameUI cropped.CopyTo(pos); // - bool result1 = GridScreen.GridEnumerator.IsScrolling(mat, scrolled, out Point2d shift); + bool result1 = GridScreen.GridEnumerator.IsScrolling(mat, scrolled, out Point2d shift, upperThreshold: 0.99); bool result2 = GridScreen.GridEnumerator.IsScrolling(mat, mat, out Point2d _); bool result3 = GridScreen.GridEnumerator.IsScrolling(mat, black, out Point2d _);