using BetterGenshinImpact.Core.Config; using BetterGenshinImpact.Core.Recognition.OCR; using BetterGenshinImpact.Core.Recognition.ONNX; using BetterGenshinImpact.Core.Recognition.OpenCv; using BetterGenshinImpact.GameTask.AutoFight.Assets; using BetterGenshinImpact.GameTask.AutoFight.Config; using BetterGenshinImpact.GameTask.Model.Area; using BetterGenshinImpact.Helpers; using Compunet.YoloV8; using Microsoft.Extensions.Logging; using OpenCvSharp; using OpenCvSharp.Extensions; using Sdcb.PaddleOCR; using System; using System.Collections.Generic; using System.Diagnostics; using System.Drawing; using System.Drawing.Imaging; using System.IO; using System.Linq; using System.Threading; using static BetterGenshinImpact.GameTask.Common.TaskControl; namespace BetterGenshinImpact.GameTask.AutoFight.Model; /// /// 战斗场景 /// public class CombatScenes : IDisposable { /// /// 当前配队 /// public Avatar[] Avatars { get; set; } = Array.Empty(); public Dictionary AvatarMap { get; set; } = []; public int AvatarCount { get; set; } private readonly YoloV8Predictor _predictor = YoloV8Builder.CreateDefaultBuilder() .UseOnnxModel(Global.Absolute(@"Assets\Model\Common\avatar_side_classify_sim.onnx")) .WithSessionOptions(BgiSessionOption.Instance.Options) .Build(); public int ExpectedTeamAvatarNum { get; private set; } = 4; /// /// 通过YOLO分类器识别队伍内角色 /// /// 完整游戏画面的捕获截图 public CombatScenes InitializeTeam(ImageRegion imageRegion) { // 优先取配置 if (!string.IsNullOrEmpty(TaskContext.Instance().Config.AutoFightConfig.TeamNames)) { InitializeTeamFromConfig(TaskContext.Instance().Config.AutoFightConfig.TeamNames); return this; } // 判断当前是否处于联机状态 List avatarSideIconRectList; List avatarIndexRectList; var pRaList = imageRegion.FindMulti(AutoFightAssets.Instance.PRa); if (pRaList.Count > 0) { var num = pRaList.Count + 1; if (num > 4) { throw new Exception("当前处于联机状态,但是队伍人数超过4人,无法识别"); } // 联机状态下判断 var onePRa = imageRegion.Find(AutoFightAssets.Instance.OnePRa); var p = "p"; if (!onePRa.IsEmpty()) { Logger.LogInformation("当前处于联机状态,且当前账号是房主,联机人数{Num}人", num); p = "1p"; } else { Logger.LogInformation("当前处于联机状态,且在别人世界中,联机人数{Num}人", num); } avatarSideIconRectList = AutoFightAssets.Instance.AvatarSideIconRectListMap[$"{p}_{num}"]; avatarIndexRectList = AutoFightAssets.Instance.AvatarIndexRectListMap[$"{p}_{num}"]; ExpectedTeamAvatarNum = avatarSideIconRectList.Count; } else { avatarSideIconRectList = AutoFightAssets.Instance.AvatarSideIconRectList; avatarIndexRectList = AutoFightAssets.Instance.AvatarIndexRectList; } // 识别队伍 var names = new string[avatarSideIconRectList.Count]; var displayNames = new string[avatarSideIconRectList.Count]; try { for (var i = 0; i < avatarSideIconRectList.Count; i++) { var ra = imageRegion.DeriveCrop(avatarSideIconRectList[i]); var pair = ClassifyAvatarCnName(ra.SrcBitmap, i + 1); names[i] = pair.Item1; if (!string.IsNullOrEmpty(pair.Item2)) { var costumeName = pair.Item2; if (AutoFightAssets.Instance.AvatarCostumeMap.TryGetValue(costumeName, out string? name)) { costumeName = name; } displayNames[i] = $"{pair.Item1}({costumeName})"; } else { displayNames[i] = pair.Item1; } } Logger.LogInformation("识别到的队伍角色:{Text}", string.Join(",", displayNames)); Avatars = BuildAvatars([.. names], null, avatarIndexRectList); AvatarMap = Avatars.ToDictionary(x => x.Name); } catch (Exception e) { Logger.LogWarning(e.Message); } return this; } public (string, string) ClassifyAvatarCnName(Bitmap src, int index) { var className = ClassifyAvatarName(src, index); var nameEn = className; var costumeName = ""; var i = className.IndexOf("Costume", StringComparison.Ordinal); if (i > 0) { nameEn = className[..i]; costumeName = className[(i + 7)..]; } var avatar = DefaultAutoFightConfig.CombatAvatarNameEnMap[nameEn]; return (avatar.Name, costumeName); } public string ClassifyAvatarName(Bitmap src, int index) { SpeedTimer speedTimer = new(); using var memoryStream = new MemoryStream(); src.Save(memoryStream, ImageFormat.Bmp); memoryStream.Seek(0, SeekOrigin.Begin); speedTimer.Record("角色侧面头像图像转换"); var result = _predictor.Classify(memoryStream); speedTimer.Record("角色侧面头像分类识别"); Debug.WriteLine($"角色侧面头像识别结果:{result}"); speedTimer.DebugPrint(); if (result.TopClass.Name.Name.StartsWith("Qin") || result.TopClass.Name.Name.Contains("Costume")) { // 降低琴和衣装角色的识别率要求 if (result.TopClass.Confidence < 0.51) { Cv2.ImWrite(@"log\avatar_side_classify_error.png", src.ToMat()); throw new Exception($"无法识别第{index}位角色,置信度{result.TopClass.Confidence},结果:{result.TopClass.Name.Name}"); } } else { if (result.TopClass.Confidence < 0.8) { Cv2.ImWrite(@"log\avatar_side_classify_error.png", src.ToMat()); throw new Exception($"无法识别第{index}位角色,置信度{result.TopClass.Confidence},结果:{result.TopClass.Name.Name}"); } } return result.TopClass.Name.Name; } private void InitializeTeamFromConfig(string teamNames) { var names = teamNames.Split([",", ","], StringSplitOptions.TrimEntries); if (names.Length != 4) { throw new Exception($"强制指定队伍角色数量不正确,必须是4个,当前{names.Length}个"); } // 别名转换为标准名称 for (var i = 0; i < names.Length; i++) { names[i] = DefaultAutoFightConfig.AvatarAliasToStandardName(names[i]); } Logger.LogInformation("强制指定队伍角色:{Text}", string.Join(",", names)); TaskContext.Instance().Config.AutoFightConfig.TeamNames = string.Join(",", names); Avatars = BuildAvatars([.. names]); AvatarMap = Avatars.ToDictionary(x => x.Name); } public bool CheckTeamInitialized() { if (Avatars.Length != ExpectedTeamAvatarNum) { return false; } return true; } private Avatar[] BuildAvatars(List names, List? nameRects = null, List? avatarIndexRectList = null) { if (avatarIndexRectList == null && ExpectedTeamAvatarNum == 4) { avatarIndexRectList = AutoFightContext.Instance.FightAssets.AvatarIndexRectList; } if (avatarIndexRectList == null) { throw new Exception("联机状态下,此方法必须传入队伍角色编号位置信息"); } AvatarCount = names.Count; var avatars = new Avatar[AvatarCount]; for (var i = 0; i < AvatarCount; i++) { var nameRect = nameRects?[i] ?? Rect.Empty; avatars[i] = new Avatar(this, names[i], i + 1, nameRect) { IndexRect = avatarIndexRectList[i] }; } return avatars; } public void BeforeTask(CancellationToken ct) { for (var i = 0; i < AvatarCount; i++) { Avatars[i].Ct = ct; } } public Avatar? SelectAvatar(string name) { return AvatarMap.GetValueOrDefault(name); } #region OCR识别队伍(已弃用) /// /// 通过OCR识别队伍内角色 /// /// 完整游戏画面的捕获截图 [Obsolete] public CombatScenes InitializeTeamOldOcr(CaptureContent content) { // 优先取配置 if (!string.IsNullOrEmpty(TaskContext.Instance().Config.AutoFightConfig.TeamNames)) { InitializeTeamFromConfig(TaskContext.Instance().Config.AutoFightConfig.TeamNames); return this; } // 剪裁出队伍区域 var teamRa = content.CaptureRectArea.DeriveCrop(AutoFightContext.Instance.FightAssets.TeamRectNoIndex); // 过滤出白色 var hsvFilterMat = OpenCvCommonHelper.InRangeHsv(teamRa.SrcMat, new Scalar(0, 0, 210), new Scalar(255, 30, 255)); // 识别队伍内角色 var result = OcrFactory.Paddle.OcrResult(hsvFilterMat); ParseTeamOcrResult(result, teamRa); return this; } [Obsolete] private void ParseTeamOcrResult(PaddleOcrResult result, ImageRegion rectArea) { List names = []; List nameRects = []; foreach (var item in result.Regions) { var name = StringUtils.ExtractChinese(item.Text); name = ErrorOcrCorrection(name); if (IsGenshinAvatarName(name)) { names.Add(name); nameRects.Add(item.Rect.BoundingRect()); } } if (names.Count != 4) { Logger.LogWarning("识别到的队伍角色数量不正确,当前识别结果:{Text}", string.Join(",", names)); } if (names.Count == 3) { // 流浪者特殊处理 // 4人以上的队伍,不支持流浪者的识别 var wanderer = rectArea.Find(AutoFightContext.Instance.FightAssets.WandererIconRa); if (wanderer.IsEmpty()) { wanderer = rectArea.Find(AutoFightContext.Instance.FightAssets.WandererIconNoActiveRa); } if (wanderer.IsEmpty()) { // 补充识别流浪者 Logger.LogWarning("二次尝试识别失败,当前识别结果:{Text}", string.Join(",", names)); } else { names.Clear(); foreach (var item in result.Regions) { var name = StringUtils.ExtractChinese(item.Text); name = ErrorOcrCorrection(name); if (IsGenshinAvatarName(name)) { names.Add(name); nameRects.Add(item.Rect.BoundingRect()); } var rect = item.Rect.BoundingRect(); if (rect.Y > wanderer.Y && wanderer.Y + wanderer.Height > rect.Y + rect.Height && !names.Contains("流浪者")) { names.Add("流浪者"); nameRects.Add(item.Rect.BoundingRect()); } } if (names.Count != 4) { Logger.LogWarning("图像识别到流浪者,但识别队内位置信息失败"); } } } Logger.LogInformation("识别到的队伍角色:{Text}", string.Join(",", names)); Avatars = BuildAvatars(names, nameRects); AvatarMap = Avatars.ToDictionary(x => x.Name); } [Obsolete] private bool IsGenshinAvatarName(string name) { if (DefaultAutoFightConfig.CombatAvatarNames.Contains(name)) { return true; } return false; } /// /// 对OCR识别结果进行纠错 /// TODO 还剩下单字名称(魈、琴)无法识别到的问题 /// /// /// [Obsolete] public string ErrorOcrCorrection(string name) { if (name.Contains("纳西")) { return "纳西妲"; } return name; } #endregion OCR识别队伍(已弃用) public void Dispose() { _predictor.Dispose(); } }