Files
better-genshin-impact/BetterGenshinImpact/GameTask/AutoFishing/AutoFishingTask.cs
Takaranoao 20fe152630 尝试修复一些ROI越界 (#2808)
* fix: 修复多处 OpenCV ROI 越界导致的断言失败

在低分辨率(如 1280x720)下,多处 Rect 坐标计算未做边界保护,
直接传入 SubMat / new Mat(mat, rect) 时触发 OpenCV ROI 断言崩溃。

修复位置:
- Behaviours.cs: fishBoxRect 计算结果钳位到图像边界,修复钓鱼任务越界
- GridScreen.cs: PostProcess 中幻影格子(插值生成)越界时直接丢弃
- ImageRegion.cs: DeriveCrop 两个重载统一加入坐标钳位与有效性校验
- GetGridIconsTask.cs: CropResizeArtifactSetFilterGridIcon X/Y 坐标加非负保护
- GeniusInvokationControl.cs: 角色区域扩展和 HP 区域 Y 偏移各加边界保护

* chore: 为 AutoFishingTask 鱼饵图标裁剪补充说明注释

* refactor: 提取 Rect 钳位逻辑为共享扩展方法 ClampTo

将 6 处重复的 ROI 钳位代码统一为 CommonExtension.ClampTo 扩展方法,
采用交集语义(坐标钳位时宽高同步缩减,不会扩大矩形)。
删除 AutoLeyLineOutcropTask 中的私有 ClampRect 方法。
2026-02-20 15:08:19 +08:00

478 lines
24 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using BehaviourTree;
using BehaviourTree.Composites;
using BehaviourTree.FluentBuilder;
using BetterGenshinImpact.Core.Recognition.OCR;
using BetterGenshinImpact.Core.Recognition.ONNX;
using BetterGenshinImpact.Core.Simulator;
using BetterGenshinImpact.GameTask.AutoFight.Assets;
using BetterGenshinImpact.GameTask.AutoFishing.Assets;
using BetterGenshinImpact.GameTask.AutoFishing.Model;
using BetterGenshinImpact.GameTask.Common;
using BetterGenshinImpact.GameTask.Common.BgiVision;
using BetterGenshinImpact.GameTask.Common.Job;
using BetterGenshinImpact.GameTask.GetGridIcons;
using BetterGenshinImpact.GameTask.Model.Area;
using BetterGenshinImpact.Helpers;
using BetterGenshinImpact.Helpers.Extensions;
using BetterGenshinImpact.View.Drawable;
using Compunet.YoloSharp;
using Fischless.WindowsInput;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Localization;
using Microsoft.Extensions.Logging;
using Microsoft.ML.OnnxRuntime;
using OpenCvSharp;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Vanara.PInvoke;
using static Vanara.PInvoke.User32;
namespace BetterGenshinImpact.GameTask.AutoFishing
{
public class AutoFishingTask : ISoloTask
{
private readonly ILogger _logger = App.GetLogger<AutoFishingTask>();
private readonly InputSimulator input = Simulation.SendInput;
public string Name => "钓鱼独立任务";
private CancellationToken _ct;
private readonly AutoFishingTaskParam param;
private readonly BgiYoloPredictor _predictor =
App.ServiceProvider.GetRequiredService<BgiOnnxFactory>().CreateYoloPredictor(BgiOnnxModel.BgiFish);
public AutoFishingTask(AutoFishingTaskParam param)
{
this.param = param;
}
public Task Start(CancellationToken ct)
{
this._ct = ct;
IOcrService ocrService = OcrFactory.Paddle;
using InferenceSession session = GridIconsAccuracyTestTask.LoadModel(out Dictionary<string, float[]> prototypes);
Blackboard blackboard = new Blackboard(_predictor, this.Sleep, AutoFishingAssets.Instance);
// @formatter:off
var behaviourTree = FluentBuilder.Create<ImageRegion>()
.Sequence("钓鱼并确保完成后退出钓鱼模式")
.MySimpleParallel("在整体超时时间内钓鱼", policy: SimpleParallelPolicy.OnlyOneMustSucceed)
.Sequence("调整视角并钓鱼")
.PushLeaf(() => new MoveViewpointDown("调整视角至俯视", blackboard, _logger, param.SaveScreenshotOnKeyTick, input))
.MySimpleParallel("找鱼20秒", policy: SimpleParallelPolicy.OnlyOneMustSucceed)
.PushLeaf(() => new TurnAround("转圈圈调整视角", blackboard, _logger, param.SaveScreenshotOnKeyTick, input))
.PushLeaf(() => new FindFishTimeout("找到鱼", 20, blackboard, _logger, param.SaveScreenshotOnKeyTick))
.End()
.PushLeaf(() => new EnterFishingMode("进入钓鱼模式", blackboard, _logger, param.SaveScreenshotOnKeyTick, input, session, prototypes, cultureInfo: param.GameCultureInfo, stringLocalizer: param.StringLocalizer))
.UntilFailed(@"\")
.Sequence("一直钓鱼直到没鱼")
.AlwaysSucceed(@"\")
.Sequence("从找鱼开始")
.PushLeaf(() => new MoveViewpointDown("调整视角至俯视", blackboard, _logger, param.SaveScreenshotOnKeyTick, input))
.MySimpleParallel("找鱼10秒", policy: SimpleParallelPolicy.OnlyOneMustSucceed)
.UntilSuccess("找鱼 + 初始状态确认")
.Sequence("-")
.PushLeaf(() => new CheckInitalState("初始状态确认", blackboard, _logger, param.SaveScreenshotOnKeyTick, input))
.PushLeaf(() => new GetFishpond("检测鱼群", blackboard, _logger, param.SaveScreenshotOnKeyTick))
.End()
.End()
.PushLeaf(() => new FindFishTimeout("确认初始状态和找到鱼", 10, blackboard, _logger, param.SaveScreenshotOnKeyTick))
.End()
.PushLeaf(() => new ChooseBait("选择鱼饵", blackboard, _logger, param.SaveScreenshotOnKeyTick, TaskContext.Instance().SystemInfo, input, session, prototypes))
.MySimpleParallel("抛竿直到成功或出错", policy: SimpleParallelPolicy.OnlyOneMustSucceed)
.UntilSuccess("重复抛竿")
.Sequence("-")
.PushLeaf(() => new MoveViewpointDown("调整视角至俯视", blackboard, _logger, param.SaveScreenshotOnKeyTick, input))
//.MySimpleParallel("举起鱼竿并抛竿", policy: SimpleParallelPolicy.OnlyOneMustSucceed)
// .PushLeaf(() => new LiftAndHold("举起鱼竿", blackboard, _logger, param.SaveScreenshotOnKeyTick, input))
.PushLeaf(() => new ThrowRod("抛竿", blackboard, param.UseTorch, _logger, param.SaveScreenshotOnKeyTick, input))
//.End()
.End()
.End()
.Do("抛竿检查", _ => (blackboard.abort || blackboard.throwRodNoTarget || blackboard.throwRodNoBaitFish) ? BehaviourStatus.Failed : BehaviourStatus.Running)
.End()
.MySimpleParallel("下杆中", SimpleParallelPolicy.OnlyOneMustSucceed)
.PushLeaf(() => new CheckThrowRod("检查抛竿结果", blackboard, _logger, param.SaveScreenshotOnKeyTick)) // todo 后面串联一个召回率高的下杆中检测方法
.PushLeaf(() => new FishBite("自动提竿", blackboard, _logger, param.SaveScreenshotOnKeyTick, input, ocrService, cultureInfo: param.GameCultureInfo, stringLocalizer: param.StringLocalizer))
.PushLeaf(() => new FishBiteTimeout("下杆超时检查", param.ThrowRodTimeOutTimeoutSeconds, _logger, param.SaveScreenshotOnKeyTick, input))
.End()
.MySimpleParallel("拉条中", policy: SimpleParallelPolicy.OnlyOneMustSucceed)
.PushLeaf(() => new CheckRaiseHook("检查提竿结果", blackboard, _logger, param.SaveScreenshotOnKeyTick))
.Sequence("拉条序列")
.PushLeaf(() => new GetFishBoxArea("等待拉条出现", blackboard, _logger, param.SaveScreenshotOnKeyTick))
.PushLeaf(() => new Fishing("钓鱼拉条", blackboard, _logger, param.SaveScreenshotOnKeyTick, input))
.End()
.End()
.End()
.End()
.Do("冒泡-终止检查", _ => blackboard.abort ? BehaviourStatus.Failed : BehaviourStatus.Succeeded)
.End()
.End()
.End()
.PushLeaf(() => new WholeProcessTimeout("检查整体超时", param.WholeProcessTimeoutSeconds, _logger, param.SaveScreenshotOnKeyTick))
.End()
.PushLeaf(() => new QuitFishingMode("退出钓鱼模式", blackboard, _logger, param.SaveScreenshotOnKeyTick, input, param.GameCultureInfo, param.StringLocalizer))
.End()
.Build();
// @formatter:on
_logger.LogInformation("→ {Text}", "自动钓鱼,启动!");
_logger.LogWarning("请不要携带任何{Msg},极有可能会误识别导致拖慢速度!", "跟宠");
_logger.LogInformation(
$"当前参数:{param.WholeProcessTimeoutSeconds}{param.ThrowRodTimeOutTimeoutSeconds}{param.FishingTimePolicy}, {param.SaveScreenshotOnKeyTick}, {param.GameCultureInfo}, {param.UseTorch}");
TaskContext.Instance().Config.AutoFishingConfig.Enabled = false;
_logger.LogInformation("全自动运行时,自动切换实时任务中的半自动钓鱼功能为关闭状态");
void tickARound()
{
blackboard.Reset();
var prevManualGc = DateTime.MinValue;
while (!ct.IsCancellationRequested)
{
if (!SystemControl.IsGenshinImpactActiveByProcess())
{
var name = SystemControl.GetActiveByProcess();
_logger.LogWarning($"当前获取焦点的窗口为: {name},不是原神,停止执行");
break;
}
using var bitmap =
TaskControl.CaptureGameImageNoRetry(TaskTriggerDispatcher.Instance().GameCapture);
if (bitmap == null)
{
_logger.LogWarning("截图失败");
continue;
}
using var content = new CaptureContent(bitmap, 0, 0);
behaviourTree.Tick(content.CaptureRectArea);
if (behaviourTree.Status != BehaviourStatus.Running)
{
_logger.LogInformation("钓鱼结束");
break;
}
if ((DateTime.Now - prevManualGc).TotalSeconds > 2)
{
GC.Collect();
prevManualGc = DateTime.Now;
}
}
}
using var ra = TaskControl.CaptureToRectArea(forceNew: true);
if (ra.FindMulti(AutoFightAssets.Instance.PRa).Count != 0)
{
_logger.LogInformation("当前处于联机状态,不使用昼夜设置");
tickARound();
}
else if (param.FishingTimePolicy == FishingTimePolicy.DontChange)
{
tickARound();
}
else
{
SetTimeTask setTimeTask = new SetTimeTask();
foreach (int hour in param.FishingTimePolicy == FishingTimePolicy.Daytime
? [7]
: (param.FishingTimePolicy == FishingTimePolicy.Nighttime ? [19] : new int[] { 7, 19 }))
{
setTimeTask.Start(hour, 0, ct).Wait(ct);
tickARound();
}
}
_logger.LogInformation("→ 钓鱼任务结束");
return Task.CompletedTask; // todo 这个行为树库不支持异步编程。。。
}
public void Sleep(int millisecondsTimeout)
{
TaskControl.Sleep(millisecondsTimeout, _ct);
}
public class WholeProcessTimeout : BaseBehaviour<ImageRegion>
{
private DateTime? timeout;
private readonly int seconds;
/// <summary>
/// 如果未超时返回运行中,超时返回成功
/// </summary>
/// <param name="name"></param>
/// <param name="seconds"></param>
public WholeProcessTimeout(string name, int seconds, ILogger logger, bool saveScreenshotOnTerminate) : base(
name, logger, saveScreenshotOnTerminate)
{
this.seconds = seconds;
}
protected override void OnInitialize()
{
logger.LogInformation($"钓鱼任务将在{seconds}秒后超时");
timeout = DateTime.Now.AddSeconds(seconds);
}
protected override BehaviourStatus Update(ImageRegion _)
{
if (DateTime.Now >= timeout)
{
logger.LogInformation($"{seconds}秒超时已到,强制结束任务");
return BehaviourStatus.Succeeded;
}
else
{
return BehaviourStatus.Running;
}
}
}
public class FindFishTimeout : BaseBehaviour<ImageRegion>
{
private readonly Blackboard blackboard;
private DateTime? timeout;
private readonly int seconds;
/// <summary>
/// 如果未超时返回运行中,超时返回失败
/// </summary>
/// <param name="name">行为名将反映在提示语中</param>
/// <param name="seconds"></param>
public FindFishTimeout(string name, int seconds, Blackboard blackboard, ILogger logger,
bool saveScreenshotOnTerminate) : base(name, logger, saveScreenshotOnTerminate)
{
this.blackboard = blackboard;
this.seconds = seconds;
}
protected override void OnInitialize()
{
timeout = DateTime.Now.AddSeconds(seconds);
}
protected override BehaviourStatus Update(ImageRegion _)
{
if (DateTime.Now >= timeout)
{
blackboard.abort = true;
logger.LogInformation($"{seconds}秒没有{Name},退出钓鱼界面");
return BehaviourStatus.Failed;
}
else
{
return BehaviourStatus.Running;
}
}
}
public class TurnAround : BaseBehaviour<ImageRegion>
{
private readonly IInputSimulator input;
private readonly Blackboard blackboard;
public TurnAround(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate,
IInputSimulator input) : base(name, logger, saveScreenshotOnTerminate)
{
this.blackboard = blackboard;
this.input = input;
}
protected override BehaviourStatus Update(ImageRegion imageRegion)
{
var result = blackboard.Predictor.Predictor.Detect(imageRegion.CacheImage);
if (result.Any())
{
Fishpond fishpond = new Fishpond(result);
logger.LogInformation("定位到鱼塘:" + string.Join('、',
fishpond.Fishes.GroupBy(f => f.FishType).Select(g => $"{g.Key.ChineseName}{g.Count()}条")));
int i = 0;
foreach (var fish in fishpond.Fishes)
{
imageRegion.Derive(fish.Rect).DrawSelf($"{fish.FishType.ChineseName}.{i++}");
}
blackboard.Sleep(1000);
VisionContext.Instance().DrawContent.ClearAll();
var oneFourthX = imageRegion.CacheImage.Width / 4;
var threeFourthX = imageRegion.CacheImage.Width * 3 / 4;
var centerY = imageRegion.CacheImage.Height / 2;
if (fishpond.FishpondRect.Left > threeFourthX)
{
Simulation.SendInput.Mouse.MoveMouseBy(100, 0);
blackboard.Sleep(100);
return BehaviourStatus.Running;
}
else if (fishpond.FishpondRect.Right < oneFourthX)
{
Simulation.SendInput.Mouse.MoveMouseBy(-100, 0);
blackboard.Sleep(100);
return BehaviourStatus.Running;
}
#region 1使2F交互键被吞
// 加入昼夜切换后使用KeyPress按S键被莫名吞掉了
// 并且发现如果原地空格跳跃后紧跟按一下S键角色会向侧后方走去
// 于是使用“按一段时间”来代替KeyPress的“按一瞬间”以求稳定的表现
Simulation.SendInput.Keyboard.KeyDown(User32.VK.VK_S);
blackboard.Sleep(100);
Simulation.SendInput.Keyboard.KeyUp(User32.VK.VK_S);
blackboard.Sleep(400);
Simulation.SendInput.Keyboard.KeyDown(User32.VK.VK_W);
blackboard.Sleep(100);
Simulation.SendInput.Keyboard.KeyUp(User32.VK.VK_W);
blackboard.Sleep(400);
blackboard.Sleep(300);
#endregion
logger.LogInformation("视角调整完毕");
return BehaviourStatus.Succeeded;
}
input.Mouse.MoveMouseBy(100, 0);
blackboard.Sleep(100);
return BehaviourStatus.Running;
}
}
private class EnterFishingMode : BaseBehaviour<ImageRegion>
{
private readonly IInputSimulator input;
private readonly Blackboard blackboard;
private readonly InferenceSession session;
private readonly Dictionary<string, float[]> prototypes;
private readonly TimeProvider timeProvider;
private DateTimeOffset? pressFWaitEndTime;
private DateTimeOffset? clickWhiteConfirmButtonWaitEndTime;
private DateTimeOffset? overallWaitEndTime;
private readonly string fishingLocalizedString;
public EnterFishingMode(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate,
IInputSimulator input, InferenceSession session, Dictionary<string, float[]> prototypes, TimeProvider? timeProvider = null, CultureInfo? cultureInfo = null, IStringLocalizer? stringLocalizer = null) : base(name,
logger, saveScreenshotOnTerminate)
{
this.blackboard = blackboard;
this.input = input;
this.session = session;
this.prototypes = prototypes;
this.timeProvider = timeProvider ?? TimeProvider.System;
this.fishingLocalizedString = stringLocalizer == null ? "钓鱼" : stringLocalizer.WithCultureGet(cultureInfo, "钓鱼");
}
protected override BehaviourStatus Update(ImageRegion imageRegion)
{
if (Status == BehaviourStatus.Ready)
{
overallWaitEndTime = timeProvider.GetLocalNow().AddSeconds(10);
return BehaviourStatus.Running;
}
if ((pressFWaitEndTime == null || pressFWaitEndTime < timeProvider.GetLocalNow()) &&
Bv.FindFAndPress(imageRegion, input.Keyboard, this.fishingLocalizedString))
{
logger.LogInformation("按下钓鱼键");
pressFWaitEndTime = timeProvider.GetLocalNow().AddSeconds(3);
return BehaviourStatus.Running;
}
else if ((clickWhiteConfirmButtonWaitEndTime == null ||
clickWhiteConfirmButtonWaitEndTime < timeProvider.GetLocalNow()) &&
Bv.ClickWhiteConfirmButton(imageRegion))
{
// 截取鱼饵图标区域(正方形,宽高均取 6.5% 的屏幕宽度)
// 最后一个参数有意用 Width 而非 Height目的是保持正方形裁剪
// 经验算在 16:9 常见分辨率720p/1080p/1440p下 Y+H 不会超出图像高度,暂不加钳位
using Mat subMat = imageRegion.SrcMat.SubMat(new Rect((int)(0.824 * imageRegion.Width), (int)(0.669 * imageRegion.Height), (int)(0.065 * imageRegion.Width), (int)(0.065 * imageRegion.Width)));
using Mat resized = subMat.Resize(new Size(125, 125));
(string predName, _) = GridIconsAccuracyTestTask.Infer(resized, this.session, this.prototypes);
if (predName.TryGetEnumValueFromDescription(out this.blackboard.selectedBait))
{
logger.LogInformation("点击开始钓鱼,当前鱼饵为{bait}", this.blackboard.selectedBait.Value.GetDescription());
}
else
{
logger.LogInformation("点击开始钓鱼,当前鱼饵未识别");
}
this.blackboard.pitchReset = true;
clickWhiteConfirmButtonWaitEndTime = timeProvider.GetLocalNow().AddSeconds(3);
return BehaviourStatus.Running;
}
if (imageRegion.Find(blackboard.AutoFishingAssets.ExitFishingButtonRo).IsEmpty())
{
if (overallWaitEndTime < timeProvider.GetLocalNow())
{
logger.LogInformation("进入钓鱼模式失败");
return BehaviourStatus.Failed;
}
else
{
return BehaviourStatus.Running;
}
}
else
{
logger.LogInformation("进入钓鱼模式");
return BehaviourStatus.Succeeded;
}
}
}
private class QuitFishingMode : BaseBehaviour<ImageRegion>
{
private readonly IInputSimulator input;
private readonly Blackboard blackboard;
private readonly string fishingLocalizedString;
public QuitFishingMode(string name, Blackboard blackboard, ILogger logger, bool saveScreenshotOnTerminate,
IInputSimulator input, CultureInfo? cultureInfo = null, IStringLocalizer? stringLocalizer = null) : base(name, logger, saveScreenshotOnTerminate)
{
this.blackboard = blackboard;
this.input = input;
this.fishingLocalizedString = stringLocalizer == null ? "钓鱼" : stringLocalizer.WithCultureGet(cultureInfo, "钓鱼");
}
protected override BehaviourStatus Update(ImageRegion imageRegion)
{
if (Status == BehaviourStatus.Ready)
{
return BehaviourStatus.Running;
}
if (Bv.FindF(imageRegion, this.fishingLocalizedString))
{
logger.LogInformation("退出完成");
return BehaviourStatus.Succeeded;
}
else if (Bv.ClickBlackConfirmButton(imageRegion))
{
logger.LogInformation("在“是否退出钓鱼?”界面点击确认");
blackboard.Sleep(1000);
}
else
{
input.Keyboard.KeyPress(VK.VK_ESCAPE);
blackboard.Sleep(2000);
}
return BehaviourStatus.Running;
}
}
}
}