using BetterGenshinImpact.Core.Config;
using BetterGenshinImpact.Core.Recognition.OCR;
using BetterGenshinImpact.Core.Recognition.ONNX;
using BetterGenshinImpact.Core.Recognition.OpenCv;
using BetterGenshinImpact.Core.Simulator;
using BetterGenshinImpact.GameTask.AutoFight.Assets;
using BetterGenshinImpact.GameTask.AutoFight.Config;
using BetterGenshinImpact.GameTask.Common;
using BetterGenshinImpact.GameTask.Common.Element.Assets;
using BetterGenshinImpact.GameTask.Model;
using BetterGenshinImpact.GameTask.Model.Area;
using BetterGenshinImpact.Helpers;
using Compunet.YoloSharp;
using Compunet.YoloSharp.Data;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using OpenCvSharp;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Linq;
using System.Threading;
namespace BetterGenshinImpact.GameTask.AutoFight.Model;
///
/// 战斗场景
///
public class CombatScenes : IDisposable
{
///
/// 当前配队
///
private Avatar[] Avatars { set; get; } = [];
public int AvatarCount => Avatars.Length;
///
/// 最近一次识别出的出战角色编号,从1开始,-1表示未识别
///
public int LastActiveAvatarIndex { get; set; } = -1;
public MultiGameStatus? CurrentMultiGameStatus { set; get; }
private readonly BgiYoloPredictor _predictor;
private readonly bool _ownsPredictor;
private readonly AutoFightAssets _autoFightAssets;
private readonly ElementAssets _elementAssets;
private readonly ILogger _logger;
private readonly ISystemInfo _systemInfo;
public CombatScenes(BgiYoloPredictor? predictor = null, AutoFightAssets? autoFightAssets = null, ILogger? logger = null, ElementAssets? elementAssets = null, ISystemInfo? systemInfo = null)
{
if (predictor == null)
{
_predictor = App.ServiceProvider.GetRequiredService().CreateYoloPredictor(BgiOnnxModel.BgiAvatarSide);
_ownsPredictor = true;
}
else
{
_predictor = predictor;
_ownsPredictor = false;
}
if (autoFightAssets == null)
{
_autoFightAssets = AutoFightAssets.Instance; // todo BaseAssets重构后直接由systemInfo构建,省去传入?
}
else
{
_autoFightAssets = autoFightAssets;
}
if (logger == null)
{
_logger = TaskControl.Logger;
}
else
{
_logger = logger;
}
if (elementAssets == null)
{
_elementAssets = ElementAssets.Instance;
}
else
{
_elementAssets = elementAssets;
}
if (systemInfo == null)
{
_systemInfo = TaskContext.Instance().SystemInfo;
}
else
{
_systemInfo = systemInfo;
}
}
public int ExpectedTeamAvatarNum { get; private set; } = 4;
///
/// 获取一个只读的Avatars
///
/// Avatars
public ReadOnlyCollection GetAvatars()
{
return Avatars.AsReadOnly();
}
///
/// 通过YOLO分类器识别队伍内角色
///
/// 完整游戏画面的捕获截图
public CombatScenes InitializeTeam(ImageRegion imageRegion, AutoFightConfig? autoFightConfig = null)
{
if (autoFightConfig == null)
{
autoFightConfig = TaskContext.Instance().Config.AutoFightConfig;
}
AssertUtils.CheckGameResolution();
// 优先取配置
if (!string.IsNullOrEmpty(autoFightConfig.TeamNames))
{
InitializeTeamFromConfig(autoFightConfig.TeamNames, autoFightConfig);
return this;
}
// 判断联机状态
CurrentMultiGameStatus = PartyAvatarSideIndexHelper.DetectedMultiGameStatus(imageRegion, _autoFightAssets, _logger);
// 队伍角色编号和侧面头像位置
var (avatarIndexRectList, avatarSideIconRectList) = PartyAvatarSideIndexHelper.GetAllIndexRects(imageRegion, CurrentMultiGameStatus, _logger, _elementAssets, _systemInfo);
ExpectedTeamAvatarNum = avatarIndexRectList.Count;
// 识别队伍
var names = new string[avatarSideIconRectList.Count];
var displayNames = new string[avatarSideIconRectList.Count];
try
{
for (var i = 0; i < avatarSideIconRectList.Count; i++)
{
var ra = imageRegion.DeriveCrop(avatarSideIconRectList[i]);
var pair = ClassifyAvatarCnName(ra.CacheImage, i + 1);
names[i] = pair.Item1;
if (!string.IsNullOrEmpty(pair.Item2))
{
var costumeName = pair.Item2;
if (_autoFightAssets.AvatarCostumeMap.TryGetValue(costumeName, out string? name))
{
costumeName = name;
}
displayNames[i] = $"{pair.Item1}({costumeName})";
}
else
{
displayNames[i] = pair.Item1;
}
}
_logger.LogInformation("识别到的队伍角色:{Text}", string.Join(",", displayNames));
Avatars = BuildAvatars([.. names], null, avatarIndexRectList, autoFightConfig);
}
catch (Exception e) // todo 此处catch把错误吞了不便排查
{
_logger.LogWarning(e.Message);
}
return this;
}
///
/// 这个个方法主要用于在切人判断有误的情况下,且能够找到预期数量的角色编号框。此时只有两种情况
/// 1. A草露进度条导致角色编号框偏移,B退队后偏移不变,C独立地图传送后偏移还原
/// 2. 地图边缘环境,导致角色编号框切人判断失效
/// 此方法必须在判定一定存在 ExpectedTeamAvatarNum 数量的 IndexRectList 的情况下才能使用
///
///
/// false:存在 IndexRectList 的情况下使用此方法,返回false的时候很有可能处于地图边缘环境下
public bool RefreshTeamAvatarIndexRectList(ImageRegion imageRegion)
{
// 只用新方法判断
try
{
var (avatarIndexRectList, _) = PartyAvatarSideIndexHelper.GetAllIndexRectsNew(imageRegion, CurrentMultiGameStatus!, _logger, _elementAssets, _systemInfo);
if (avatarIndexRectList.Count != ExpectedTeamAvatarNum)
{
_logger.LogWarning("重新识别到的队伍角色数量与之前不一致,之前{Old}个,现在{New}个", ExpectedTeamAvatarNum, avatarIndexRectList.Count);
return false;
}
for (var i = 0; i < ExpectedTeamAvatarNum; i++)
{
Avatars[i].IndexRect = avatarIndexRectList[i];
}
return true;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "使用新方法获取角色编号位置失败");
_logger.LogWarning("[重新识别角色编号位置]使用新方法获取角色编号位置失败,原因:" + ex.Message);
return false;
}
}
// public static List FindAvatarIndexRectList(ImageRegion imageRegion)
// {
// var i1 = imageRegion.Find(ElementAssets.Instance.Index1);
// var i2 = imageRegion.Find(ElementAssets.Instance.Index2);
// var i3 = imageRegion.Find(ElementAssets.Instance.Index3);
// var i4 = imageRegion.Find(ElementAssets.Instance.Index4);
// var curr = imageRegion.Find(ElementAssets.Instance.CurrentAvatarThreshold);
// // Debug.WriteLine($"i1:{i1.X},{i1.Y},{i1.Width},{i1.Height}; i2:{i2.X},{i2.Y},{i2.Width},{i2.Height}; i3:{i3.X},{i3.Y},{i3.Width},{i3.Height}; i4:{i4.X},{i4.Y},{i4.Width},{i4.Height}; curr:{curr.X},{curr.Y},{curr.Width},{curr.Height}");
// return null;
// }
public (string, string) ClassifyAvatarCnName(Image img, int index)
{
var className = ClassifyAvatarName(img, index);
var nameEn = className;
var costumeName = "";
var i = className.IndexOf("Costume", StringComparison.Ordinal);
if (i > 0)
{
nameEn = className[..i];
costumeName = className[(i + 7)..];
}
var avatar = DefaultAutoFightConfig.CombatAvatarNameEnMap[nameEn];
return (avatar.Name, costumeName);
}
public string ClassifyAvatarName(Image img, int index)
{
SpeedTimer speedTimer = new();
speedTimer.Record("角色侧面头像图像转换");
var result = _predictor.Predictor.Classify(img);
speedTimer.Record("角色侧面头像分类识别");
Debug.WriteLine($"角色侧面头像识别结果:{result}");
speedTimer.DebugPrint();
var topClass = result.GetTopClass();
if (topClass.Name.Name.StartsWith("Qin") || topClass.Name.Name.Contains("Costume"))
{
// 降低琴和衣装角色的识别率要求
if (topClass.Confidence < 0.51)
{
img.SaveAsPng(Global.Absolute($@"log\avatar_side_classify_error.png"));
throw new Exception(
$"无法识别第{index}位角色,置信度{topClass.Confidence:F1},结果:{topClass.Name.Name}。请重新阅读 BetterGI 文档中的《快速上手》!");
}
}
else
{
if (topClass.Confidence < 0.7)
{
img.SaveAsPng(Global.Absolute($@"log\avatar_side_classify_error.png"));
throw new Exception(
$"无法识别第{index}位角色,置信度{topClass.Confidence:F1},结果:{topClass.Name.Name}。请重新阅读 BetterGI 文档中的《快速上手》!");
}
}
return topClass.Name.Name;
}
private void InitializeTeamFromConfig(string teamNames, AutoFightConfig autoFightConfig)
{
var names = teamNames.Split([",", ","], StringSplitOptions.TrimEntries);
if (names.Length != 4)
{
throw new Exception($"强制指定队伍角色数量不正确,必须是4个,当前{names.Length}个");
}
// 别名转换为标准名称
for (var i = 0; i < names.Length; i++)
{
names[i] = DefaultAutoFightConfig.AvatarAliasToStandardName(names[i]);
}
_logger.LogInformation("强制指定队伍角色:{Text}", string.Join(",", names));
autoFightConfig.TeamNames = string.Join(",", names);
Avatars = BuildAvatars([.. names], autoFightConfig: autoFightConfig);
}
public bool CheckTeamInitialized()
{
if (Avatars.Length != ExpectedTeamAvatarNum)
{
return false;
}
return true;
}
private Avatar[] BuildAvatars(List names, List? nameRects = null, List? avatarIndexRectList = null, AutoFightConfig? autoFightConfig = null)
{
if (autoFightConfig == null)
{
autoFightConfig = TaskContext.Instance().Config.AutoFightConfig;
}
var cdConfig = autoFightConfig.ActionSchedulerByCd;
if (avatarIndexRectList == null && ExpectedTeamAvatarNum == 4)
{
avatarIndexRectList = _autoFightAssets.AvatarIndexRectList;
}
if (avatarIndexRectList == null)
{
throw new Exception("联机状态下,此方法必须传入队伍角色编号位置信息");
}
var namesCount = names.Count;
var avatars = new Avatar[namesCount];
for (var i = 0; i < namesCount; i++)
{
var nameRect = nameRects?[i] ?? default;
// 根据手动写的出招表来优化CD
var cd = Avatar.ParseActionSchedulerByCd(names[i], cdConfig);
avatars[i] = new Avatar(this, names[i], i + 1, nameRect, cd ?? -1)
{
IndexRect = avatarIndexRectList[i]
};
}
return avatars;
}
///
/// 更新角色手动设置的CD
///
/// 配置字符串
/// 返回配置中有效的角色名
public List UpdateActionSchedulerByCd(string cdConfig)
{
if (string.IsNullOrEmpty(cdConfig))
{
return [];
}
List names = [];
foreach (var t in Avatars)
{
var mCd = Avatar.ParseActionSchedulerByCd(t.Name, cdConfig);
// 手动cd不为0,不是麦当劳不是0
if (mCd is null) continue;
t.ManualSkillCd = (double)mCd;
names.Add(t.Name);
}
return names;
}
public void BeforeTask(CancellationToken ct)
{
for (var i = 0; i < AvatarCount; i++)
{
Avatars[i].Ct = ct;
}
}
public void AfterTask()
{
// 释放所有按键
Simulation.ReleaseAllKey();
var mwk = SelectAvatar("玛薇卡");
if (mwk != null)
{
foreach (var avatar in Avatars)
{
if (avatar.Name != "玛薇卡")
{
avatar.Switch();
}
}
}
}
public Avatar? SelectAvatar(string name)
{
return Avatars.FirstOrDefault(avatar => avatar.Name.Equals(name));
}
///
/// 使用编号切换角色
///
/// 从1开始
///
public Avatar SelectAvatar(int avatarIndex)
{
if (avatarIndex < 1 || avatarIndex > AvatarCount)
{
_logger.LogError("切换角色编号错误,当前角色数量{Count},编号{Index}", AvatarCount, avatarIndex);
throw new Exception("不存在的角色编号");
}
return Avatars[avatarIndex - 1];
}
///
/// 获取当前出战角色名
/// 不考虑重新刷新编号框位置
/// 不推荐使用
///
///
///
///
///
public string? CurrentAvatar(bool force = false, ImageRegion? region = null,
CancellationToken ct = default)
{
if (!force && LastActiveAvatarIndex > 0)
{
return Avatars[LastActiveAvatarIndex - 1].Name;
}
var imageRegion = region ?? TaskControl.CaptureToRectArea();
var rectArray = Avatars.Select(t => t.IndexRect).ToArray();
int index = PartyAvatarSideIndexHelper.GetAvatarIndexIsActiveWithContext(imageRegion, rectArray, new AvatarActiveCheckContext());
if (index > 0)
{
LastActiveAvatarIndex = index;
}
return Avatars[LastActiveAvatarIndex - 1].Name;
}
///
/// 推荐使用
/// 失败后自动刷新编号框位置
///
/// l
///
///
public int GetActiveAvatarIndex(ImageRegion imageRegion, AvatarActiveCheckContext context)
{
var rectArray = Avatars.Select(t => t.IndexRect).ToArray();
int index = PartyAvatarSideIndexHelper.GetAvatarIndexIsActiveWithContext(imageRegion, rectArray, context);
if (index > 0)
{
LastActiveAvatarIndex = index;
return index;
}
else
{
// 多次识别失败则尝试刷新角色编号位置
// 应对草露问题
if (context.TotalCheckFailedCount > 3)
{
// 失败多次,识别是否存在满足预期的编号框
if (PartyAvatarSideIndexHelper.CountIndexRect(imageRegion) == Avatars.Length)
{
bool res = RefreshTeamAvatarIndexRectList(imageRegion);
_logger.LogWarning("多次识别出战角色失败,尝试刷新角色编号位置(处理草露问题),刷新结果:{Result}", res ? "成功" : "失败");
if (res)
{
context.TotalCheckFailedCount = 0;
}
}
}
}
return -1;
}
#region OCR识别队伍(已弃用)
///
/// 通过OCR识别队伍内角色
///
/// 完整游戏画面的捕获截图
[Obsolete]
public CombatScenes InitializeTeamOldOcr(CaptureContent content)
{
// 优先取配置
if (!string.IsNullOrEmpty(TaskContext.Instance().Config.AutoFightConfig.TeamNames))
{
InitializeTeamFromConfig(TaskContext.Instance().Config.AutoFightConfig.TeamNames, TaskContext.Instance().Config.AutoFightConfig);
return this;
}
// 剪裁出队伍区域
var teamRa = content.CaptureRectArea.DeriveCrop(_autoFightAssets.TeamRectNoIndex);
// 过滤出白色
var hsvFilterMat =
OpenCvCommonHelper.InRangeHsv(teamRa.SrcMat, new Scalar(0, 0, 210), new Scalar(255, 30, 255));
// 识别队伍内角色
var result = OcrFactory.Paddle.OcrResult(hsvFilterMat);
ParseTeamOcrResult(result, teamRa);
return this;
}
[Obsolete]
private void ParseTeamOcrResult(OcrResult result, ImageRegion rectArea)
{
List names = [];
List nameRects = [];
foreach (var item in result.Regions)
{
var name = StringUtils.ExtractChinese(item.Text);
name = ErrorOcrCorrection(name);
if (IsGenshinAvatarName(name))
{
names.Add(name);
nameRects.Add(item.Rect.BoundingRect());
}
}
if (names.Count != 4)
{
_logger.LogWarning("识别到的队伍角色数量不正确,当前识别结果:{Text}", string.Join(",", names));
}
if (names.Count == 3)
{
// 流浪者特殊处理
// 4人以上的队伍,不支持流浪者的识别
var wanderer = rectArea.Find(_autoFightAssets.WandererIconRa);
if (wanderer.IsEmpty())
{
wanderer = rectArea.Find(_autoFightAssets.WandererIconNoActiveRa);
}
if (wanderer.IsEmpty())
{
// 补充识别流浪者
_logger.LogWarning("二次尝试识别失败,当前识别结果:{Text}", string.Join(",", names));
}
else
{
names.Clear();
foreach (var item in result.Regions)
{
var name = StringUtils.ExtractChinese(item.Text);
name = ErrorOcrCorrection(name);
if (IsGenshinAvatarName(name))
{
names.Add(name);
nameRects.Add(item.Rect.BoundingRect());
}
var rect = item.Rect.BoundingRect();
if (rect.Y > wanderer.Y && wanderer.Y + wanderer.Height > rect.Y + rect.Height &&
!names.Contains("流浪者"))
{
names.Add("流浪者");
nameRects.Add(item.Rect.BoundingRect());
}
}
if (names.Count != 4)
{
_logger.LogWarning("图像识别到流浪者,但识别队内位置信息失败");
}
}
}
_logger.LogInformation("识别到的队伍角色:{Text}", string.Join(",", names));
Avatars = BuildAvatars(names, nameRects);
}
[Obsolete]
private bool IsGenshinAvatarName(string name)
{
if (DefaultAutoFightConfig.CombatAvatarNames.Contains(name))
{
return true;
}
return false;
}
///
/// 对OCR识别结果进行纠错
/// TODO 还剩下单字名称(魈、琴)无法识别到的问题
///
///
///
[Obsolete]
public string ErrorOcrCorrection(string name)
{
if (name.Contains("纳西"))
{
return "纳西妲";
}
return name;
}
#endregion OCR识别队伍(已弃用)
public void Dispose()
{
if (_ownsPredictor)
{
_predictor.Dispose();
}
}
}