using BetterGenshinImpact.Core.Config;
using BetterGenshinImpact.Core.Recognition.OCR;
using BetterGenshinImpact.Core.Recognition.OpenCv;
using BetterGenshinImpact.Core.Simulator;
using BetterGenshinImpact.GameTask.AutoSkip.Assets;
using BetterGenshinImpact.GameTask.AutoSkip.Model;
using BetterGenshinImpact.GameTask.Common;
using BetterGenshinImpact.GameTask.Common.Element.Assets;
using BetterGenshinImpact.GameTask.Model;
using BetterGenshinImpact.Helpers;
using BetterGenshinImpact.Service;
using BetterGenshinImpact.View.Drawable;
using Microsoft.Extensions.Logging;
using OpenCvSharp;
using Sdcb.PaddleOCR;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using System.Windows.Forms;
using Vanara.PInvoke;
namespace BetterGenshinImpact.GameTask.AutoSkip;
///
/// 自动剧情有选项点击
///
public class AutoSkipTrigger : ITaskTrigger
{
private readonly ILogger _logger = App.GetLogger();
public string Name => "自动剧情";
public bool IsEnabled { get; set; }
public int Priority => 20;
public bool IsExclusive => false;
private readonly AutoSkipAssets _autoSkipAssets;
private readonly AutoSkipConfig _config;
///
/// 不自动点击的选项,优先级低于橙色文字点击
///
private List _defaultPauseList = new();
///
/// 不自动点击的选项
///
private List _pauseList = new();
///
/// 优先自动点击的选项
///
private List _selectList = new();
public AutoSkipTrigger()
{
_autoSkipAssets = AutoSkipAssets.Instance;
_config = TaskContext.Instance().Config.AutoSkipConfig;
}
public void Init()
{
IsEnabled = TaskContext.Instance().Config.AutoSkipConfig.Enabled;
try
{
var defaultPauseListJson = Global.ReadAllTextIfExist(@"User\AutoSkip\default_pause_options.json");
if (!string.IsNullOrEmpty(defaultPauseListJson))
{
_defaultPauseList = JsonSerializer.Deserialize>(defaultPauseListJson, ConfigService.JsonOptions) ?? new List();
}
}
catch (Exception e)
{
_logger.LogError(e, "读取自动剧情默认暂停点击关键词列表失败");
MessageBox.Show("读取自动剧情默认暂停点击关键词列表失败,请确认修改后的自动剧情默认暂停点击关键词内容格式是否正确!", "错误", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
try
{
var pauseListJson = Global.ReadAllTextIfExist(@"User\AutoSkip\pause_options.json");
if (!string.IsNullOrEmpty(pauseListJson))
{
_pauseList = JsonSerializer.Deserialize>(pauseListJson, ConfigService.JsonOptions) ?? new List();
}
}
catch (Exception e)
{
_logger.LogError(e, "读取自动剧情暂停点击关键词列表失败");
MessageBox.Show("读取自动剧情暂停点击关键词列表失败,请确认修改后的自动剧情暂停点击关键词内容格式是否正确!", "错误", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
try
{
var selectListJson = Global.ReadAllTextIfExist(@"User\AutoSkip\select_options.json");
if (!string.IsNullOrEmpty(selectListJson))
{
_selectList = JsonSerializer.Deserialize>(selectListJson, ConfigService.JsonOptions) ?? new List();
}
}
catch (Exception e)
{
_logger.LogError(e, "读取自动剧情优先点击选项列表失败");
MessageBox.Show("读取自动剧情优先点击选项列表失败,请确认修改后的自动剧情优先点击选项内容格式是否正确!", "错误", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
///
/// 上一次播放中的帧
///
private DateTime _prevPlayingTime = DateTime.MinValue;
private DateTime _prevExecute = DateTime.MinValue;
private DateTime _prevHangoutExecute = DateTime.MinValue;
private DateTime _prevGetDailyRewardsTime = DateTime.MinValue;
private DateTime _prevClickTime = DateTime.MinValue;
public void OnCapture(CaptureContent content)
{
if ((DateTime.Now - _prevExecute).TotalMilliseconds <= 200)
{
return;
}
_prevExecute = DateTime.Now;
VisionContext.Instance().DrawContent.RemoveRect("HangoutIcon");
GetDailyRewardsEsc(_config, content);
// 找左上角剧情自动的按钮
using var foundRectArea = content.CaptureRectArea.Find(_autoSkipAssets.StopAutoButtonRo);
var isPlaying = !foundRectArea.IsEmpty(); // 播放中
// 播放中图标消失3s内OCR判断文字
if (!isPlaying && (DateTime.Now - _prevPlayingTime).TotalSeconds <= 5)
{
// 找播放中的文字
content.CaptureRectArea.Find(_autoSkipAssets.PlayingTextRo, _ => { isPlaying = true; });
if (!isPlaying)
{
var textRa = content.CaptureRectArea.Crop(_autoSkipAssets.PlayingTextRo.RegionOfInterest);
// 过滤出白色
var hsvFilterMat = OpenCvCommonHelper.InRangeHsv(textRa.SrcMat, new Scalar(0, 0, 170), new Scalar(255, 80, 245));
var result = OcrFactory.Paddle.Ocr(hsvFilterMat);
if (result.Contains("播") || result.Contains("番") || result.Contains("放") || result.Contains("中") || result.Contains("潘") || result.Contains("故"))
{
VisionContext.Instance().DrawContent.PutRect("PlayingText", textRa.ConvertRelativePositionToCaptureArea().ToRectDrawable());
isPlaying = true;
}
}
if (!isPlaying)
{
// 关闭弹出页
ClosePopupPage(content);
// 自动剧情点击3s内判断
if ((DateTime.Now - _prevPlayingTime).TotalMilliseconds < 3000)
{
// 提交物品
if (SubmitGoods(content))
{
return;
}
}
}
}
else
{
VisionContext.Instance().DrawContent.RemoveRect("PlayingText");
}
if (isPlaying)
{
_prevPlayingTime = DateTime.Now;
if (TaskContext.Instance().Config.AutoSkipConfig.QuicklySkipConversationsEnabled)
{
Simulation.SendInputEx.Keyboard.KeyPress(User32.VK.VK_SPACE);
}
// 对话选项选择
var hasOption = ChatOptionChoose(content);
// 邀约选项选择 1s 1次
if (_config.AutoHangoutEventEnabled && !hasOption)
{
if ((DateTime.Now - _prevHangoutExecute).TotalMilliseconds < 1000)
{
return;
}
_prevHangoutExecute = DateTime.Now;
HangoutOptionChoose(content);
}
}
else
{
ClickBlackGameScreen(content);
}
}
///
/// 黑屏点击判断
///
///
///
private bool ClickBlackGameScreen(CaptureContent content)
{
// 黑屏剧情要点击鼠标(多次) 几乎全黑的时候不用点击
using var grayMat = new Mat(content.CaptureRectArea.SrcGreyMat, new Rect(0, content.CaptureRectArea.SrcGreyMat.Height / 3, content.CaptureRectArea.SrcGreyMat.Width, content.CaptureRectArea.SrcGreyMat.Height / 3));
var blackCount = OpenCvCommonHelper.CountGrayMatColor(grayMat, 0);
var rate = blackCount * 1d / (grayMat.Width * grayMat.Height);
if (rate is >= 0.5 and < 0.98999)
{
Simulation.SendInputEx.Mouse.LeftButtonClick();
if ((DateTime.Now - _prevClickTime).TotalMilliseconds > 1000)
{
_logger.LogInformation("自动剧情:{Text} 比例 {Rate}", "点击黑屏", rate.ToString("F"));
}
_prevClickTime = DateTime.Now;
return true;
}
return false;
}
private void HangoutOptionChoose(CaptureContent content)
{
var selectedRects = MatchTemplateHelper.MatchOnePicForOnePic(content.CaptureRectArea.SrcGreyMat, _autoSkipAssets.HangoutSelectedMat);
var unselectedRects = MatchTemplateHelper.MatchOnePicForOnePic(content.CaptureRectArea.SrcGreyMat, _autoSkipAssets.HangoutUnselectedMat);
if (selectedRects.Count > 0 || unselectedRects.Count > 0)
{
var captureArea = TaskContext.Instance().SystemInfo.CaptureAreaRect;
var assetScale = TaskContext.Instance().SystemInfo.AssetScale;
var clickOffset = new ClickOffset(captureArea.X, captureArea.Y, assetScale);
// 识别结果显示在遮罩上
var drawList = selectedRects.Concat(unselectedRects).Select(rect => rect.ToRectDrawable()).ToList();
VisionContext.Instance().DrawContent.PutOrRemoveRectList("HangoutIcon", drawList);
List hangoutOptionList =
[
.. selectedRects.Select(selectedRect => new HangoutOption(selectedRect, true)),
.. unselectedRects.Select(unselectedRect => new HangoutOption(unselectedRect, false)),
];
// 只有一个选项直接点击
// if (hangoutOptionList.Count == 1)
// {
// hangoutOptionList[0].Click(clickOffset);
// AutoHangoutSkipLog("点击唯一邀约选项");
// return;
// }
hangoutOptionList = hangoutOptionList.Where(hangoutOption => hangoutOption.TextRect != Rect.Empty).ToList();
if (hangoutOptionList.Count == 0)
{
return;
}
// OCR识别选项文字
foreach (var hangoutOption in hangoutOptionList)
{
using var textMat = new Mat(content.CaptureRectArea.SrcGreyMat, hangoutOption.TextRect);
var text = OcrFactory.Paddle.Ocr(textMat);
hangoutOption.OptionTextSrc = StringUtils.RemoveAllEnter(text);
}
// todo 根据文字内容决定停留还是自动点击
// 这个OCR好像不太准确
// 没有停留的选项 优先选择未点击的的选项
foreach (var hangoutOption in hangoutOptionList)
{
if (!hangoutOption.IsSelected)
{
hangoutOption.Click(clickOffset);
AutoHangoutSkipLog(hangoutOption.OptionTextSrc);
return;
}
}
// 没有未点击的选项 选择第一个已点击选项
hangoutOptionList[0].Click(clickOffset);
AutoHangoutSkipLog(hangoutOptionList[0].OptionTextSrc);
VisionContext.Instance().DrawContent.RemoveRect("HangoutIcon");
}
}
///
/// 获取橙色选项的文字
///
///
///
///
///
[Obsolete]
private string GetOrangeOptionText(Mat captureMat, RectArea foundIconRectArea, int chatOptionTextWidth)
{
var textRect = new Rect(foundIconRectArea.X + foundIconRectArea.Width, foundIconRectArea.Y, chatOptionTextWidth, foundIconRectArea.Height);
using var mat = new Mat(captureMat, textRect);
// 只提取橙色
using var bMat = OpenCvCommonHelper.Threshold(mat, new Scalar(247, 198, 50), new Scalar(255, 204, 54));
// Cv2.ImWrite("log/每日委托.png", bMat);
var whiteCount = OpenCvCommonHelper.CountGrayMatColor(bMat, 255);
var rate = whiteCount * 1.0 / (bMat.Width * bMat.Height);
if (rate < 0.06)
{
Debug.WriteLine($"识别到橙色文字区域占比:{rate}");
return string.Empty;
}
var text = OcrFactory.Paddle.Ocr(bMat);
return text;
}
private bool IsOrangeOption(Mat textMat)
{
// 只提取橙色
// Cv2.ImWrite($"log/text{DateTime.Now:yyyyMMddHHmmssffff}.png", textMat);
using var bMat = OpenCvCommonHelper.Threshold(textMat, new Scalar(247, 198, 50), new Scalar(255, 204, 54));
var whiteCount = OpenCvCommonHelper.CountGrayMatColor(bMat, 255);
var rate = whiteCount * 1.0 / (bMat.Width * bMat.Height);
Debug.WriteLine($"识别到橙色文字区域占比:{rate}");
if (rate > 0.06)
{
return true;
}
return false;
}
///
/// 领取每日委托奖励 后 10s 寻找原石是否出现,出现则按下esc
///
private void GetDailyRewardsEsc(AutoSkipConfig config, CaptureContent content)
{
if (!config.AutoGetDailyRewardsEnabled)
{
return;
}
if ((DateTime.Now - _prevGetDailyRewardsTime).TotalSeconds > 10)
{
return;
}
content.CaptureRectArea.Find(_autoSkipAssets.PrimogemRo, primogemRa =>
{
Thread.Sleep(100);
Simulation.SendInputEx.Keyboard.KeyPress(User32.VK.VK_ESCAPE);
_prevGetDailyRewardsTime = DateTime.MinValue;
primogemRa.Dispose();
});
}
private readonly Regex _enRegex = new(@"^[a-zA-Z]+$");
///
/// 新的对话选项选择
///
/// 返回 true 表示存在对话选项,但是不一定点击了
///
private bool ChatOptionChoose(CaptureContent content)
{
var captureArea = TaskContext.Instance().SystemInfo.CaptureAreaRect;
var assetScale = TaskContext.Instance().SystemInfo.AssetScale;
// 感叹号识别 遇到直接点击
var exclamationIconRa = content.CaptureRectArea.Find(_autoSkipAssets.ExclamationIconRo);
if (!exclamationIconRa.IsEmpty())
{
TaskControl.Sleep(_config.AfterChooseOptionSleepDelay);
exclamationIconRa.ClickCenter();
AutoSkipLog("点击感叹号选项");
exclamationIconRa.Dispose();
return true;
}
// 气泡识别
var chatOptionResultList = MatchTemplateHelper.MatchOnePicForOnePic(content.CaptureRectArea.SrcGreyMat[_autoSkipAssets.OptionRoi], _autoSkipAssets.OptionIconRo.TemplateImageGreyMat);
if (chatOptionResultList.Count > 0)
{
// 第一个元素就是最下面的
chatOptionResultList = chatOptionResultList.OrderByDescending(x => x.Y).ToList();
// 通过最下面的气泡框来文字识别
var lowest = chatOptionResultList[0];
var ocrRect = new Rect(_autoSkipAssets.OptionRoi.X + (int)(lowest.X + lowest.Width + 8 * assetScale), 0,
(int)(535 * assetScale), (int)(lowest.Y + lowest.Height + 30 * assetScale));
using var ocrMat = new Mat(content.CaptureRectArea.SrcGreyMat, ocrRect);
// Cv2.ImWrite("log/ocrMat.png", ocrMat);
var ocrRes = OcrFactory.Paddle.OcrResult(ocrMat);
// 删除为空的结果 和 纯英文的结果
var rs = new List();
foreach (var item in ocrRes.Regions)
{
if (string.IsNullOrEmpty(item.Text) || (item.Text.Length < 5 && _enRegex.IsMatch(item.Text)))
{
continue;
}
rs.Add(item);
}
if (rs.Count > 0)
{
var clickOffset = new ClickOffset(captureArea.X + ocrRect.X, captureArea.Y + ocrRect.Y, assetScale);
// 用户自定义关键词 匹配
foreach (var item in rs)
{
// 选择关键词
if (_selectList.Any(s => item.Text.Contains(s)))
{
ClickOcrRegion(clickOffset, item);
return true;
}
// 不选择关键词
if (_pauseList.Any(s => item.Text.Contains(s)))
{
return true;
}
}
// 橙色选项
foreach (var item in rs)
{
var textOcrRect = item.Rect.BoundingRect();
var textRect = new Rect(ocrRect.X + textOcrRect.X, ocrRect.Y + textOcrRect.Y, textOcrRect.Width, textOcrRect.Height);
if (textRect.X < 0 || textRect.Y < 0 || textRect.Width > content.CaptureRectArea.SrcMat.Width || textRect.Height > content.CaptureRectArea.SrcMat.Height)
{
Debug.WriteLine($"识别到的文字区域超出正常范围:{textOcrRect}");
_logger.LogDebug("识别到的文字区域超出正常范围:{TextOcrRect}", textOcrRect);
continue;
}
var textMat = new Mat(content.CaptureRectArea.SrcMat, textRect);
if (IsOrangeOption(textMat))
{
if (item.Text.Contains("每日委托"))
{
ClickOcrRegion(clickOffset, item);
_prevGetDailyRewardsTime = DateTime.Now; // 记录领取时间
}
else if (item.Text.Contains("探索派遣"))
{
ClickOcrRegion(clickOffset, item);
Thread.Sleep(800); // 等待探索派遣界面打开
new OneKeyExpeditionTask().Run(_autoSkipAssets);
}
else
{
ClickOcrRegion(clickOffset, item);
}
return true;
}
}
// 默认不选择关键词
foreach (var item in rs)
{
// 不选择关键词
if (_defaultPauseList.Any(s => item.Text.Contains(s)))
{
return true;
}
}
// 最后,选择默认选项
var clickRegion = rs[^1];
if (_config.ClickFirstOptionEnabled)
{
clickRegion = rs[0];
}
ClickOcrRegion(clickOffset, clickRegion);
AutoSkipLog(clickRegion.Text);
}
else
{
var clickRect = lowest;
if (_config.ClickFirstOptionEnabled)
{
clickRect = chatOptionResultList[^1];
}
// 没OCR到文字,直接选择气泡选项
TaskControl.Sleep(_config.AfterChooseOptionSleepDelay);
var clickOffset = new ClickOffset(captureArea.X + _autoSkipAssets.OptionRoi.X, captureArea.Y + _autoSkipAssets.OptionRoi.Y, assetScale);
clickOffset.ClickWithoutScale(clickRect.X + clickRect.Width / 2, clickRect.Y + clickRect.Height / 2);
var msg = _config.ClickFirstOptionEnabled ? "第一个" : "最后一个";
AutoSkipLog($"点击{msg}气泡选项");
}
return true;
}
return false;
}
private void ClickOcrRegion(ClickOffset clickOffset, PaddleOcrResultRegion clickRegion)
{
TaskControl.Sleep(_config.AfterChooseOptionSleepDelay);
clickOffset.ClickWithoutScale(clickRegion.Rect.Center.X, clickRegion.Rect.Center.Y);
AutoSkipLog(clickRegion.Text);
}
private void AutoHangoutSkipLog(string text)
{
if ((DateTime.Now - _prevClickTime).TotalMilliseconds > 1000)
{
_logger.LogInformation("自动邀约:{Text}", text);
}
_prevClickTime = DateTime.Now;
}
private void AutoSkipLog(string text)
{
if (text.Contains("每日委托") || text.Contains("探索派遣"))
{
_logger.LogInformation("自动剧情:{Text}", text);
}
else if ((DateTime.Now - _prevClickTime).TotalMilliseconds > 1000)
{
_logger.LogInformation("自动剧情:{Text}", text);
}
_prevClickTime = DateTime.Now;
}
///
/// 关闭弹出页
///
///
private void ClosePopupPage(CaptureContent content)
{
content.CaptureRectArea.Find(_autoSkipAssets.PageCloseRo, pageCloseRoRa =>
{
pageCloseRoRa.ClickCenter();
AutoSkipLog("关闭弹出页");
pageCloseRoRa.Dispose();
});
}
private bool SubmitGoods(CaptureContent content)
{
var exclamationRa = content.CaptureRectArea.Find(_autoSkipAssets.SubmitExclamationIconRo);
if (!exclamationRa.IsEmpty())
{
// var rects = MatchTemplateHelper.MatchOnePicForOnePic(content.CaptureRectArea.SrcMat.CvtColor(ColorConversionCodes.BGRA2BGR),
// _autoSkipAssets.SubmitGoodsMat, TemplateMatchModes.SqDiffNormed, null, 0.9, 4);
var rects = ContoursHelper.FindSpecifyColorRects(content.CaptureRectArea.SrcMat, new Scalar(233, 229, 220), 100, 20);
if (rects.Count == 0)
{
return false;
}
// 画矩形并保存
// foreach (var rect in rects)
// {
// Cv2.Rectangle(content.CaptureRectArea.SrcMat, rect, Scalar.Red, 1);
// }
// Cv2.ImWrite("log/提交物品.png", content.CaptureRectArea.SrcMat);
var captureArea = TaskContext.Instance().SystemInfo.CaptureAreaRect;
var assetScale = TaskContext.Instance().SystemInfo.AssetScale;
var clickOffset = new ClickOffset(captureArea.X, captureArea.Y, assetScale);
for (var i = 0; i < rects.Count; i++)
{
clickOffset.ClickWithoutScale(rects[i].X + rects[i].Width / 2, rects[i].Y + rects[i].Height / 2);
_logger.LogInformation("提交物品:{Text}", "1. 选择物品" + i);
TaskControl.Sleep(800);
content = TaskControl.CaptureToContent();
var btnBlackConfirmRa = content.CaptureRectArea.Find(ElementAssets.Instance.BtnBlackConfirm);
if (!btnBlackConfirmRa.IsEmpty())
{
btnBlackConfirmRa.ClickCenter();
_logger.LogInformation("提交物品:{Text}", "2. 放入" + i);
TaskControl.Sleep(200);
}
}
TaskControl.Sleep(500);
content = TaskControl.CaptureToContent();
var btnWhiteConfirmRa = content.CaptureRectArea.Find(ElementAssets.Instance.BtnWhiteConfirm);
if (!btnWhiteConfirmRa.IsEmpty())
{
btnWhiteConfirmRa.ClickCenter();
_logger.LogInformation("提交物品:{Text}", "3. 交付");
VisionContext.Instance().DrawContent.ClearAll();
}
// 最多4个物品 现在就支持一个
// var prevGoodsRect = Rect.Empty;
// for (var i = 1; i <= 4; i++)
// {
// // 不断的截取出右边的物品
// TaskControl.Sleep(200);
// content = TaskControl.CaptureToContent();
// var gameArea = content.CaptureRectArea;
// if (prevGoodsRect != Rect.Empty)
// {
// var r = content.CaptureRectArea.ToRect();
// var newX = prevGoodsRect.X + prevGoodsRect.Width;
// gameArea = content.CaptureRectArea.Crop(new Rect(newX, 0, r.Width - newX, r.Height));
// Cv2.ImWrite($"log/物品{i}.png", gameArea.SrcMat);
// }
//
// var goods = gameArea.Find(_autoSkipAssets.SubmitGoodsRo);
// if (!goods.IsEmpty())
// {
// prevGoodsRect = goods.ConvertRelativePositionToCaptureArea();
// goods.ClickCenter();
// _logger.LogInformation("提交物品:{Text}", "1. 选择物品" + i);
//
// TaskControl.Sleep(800);
// content = TaskControl.CaptureToContent();
//
// var btnBlackConfirmRa = content.CaptureRectArea.Find(ElementAssets.Instance().BtnBlackConfirm);
// if (!btnBlackConfirmRa.IsEmpty())
// {
// btnBlackConfirmRa.ClickCenter();
// _logger.LogInformation("提交物品:{Text}", "2. 放入" + i);
//
// TaskControl.Sleep(800);
// content = TaskControl.CaptureToContent();
//
// btnBlackConfirmRa = content.CaptureRectArea.Find(ElementAssets.Instance().BtnBlackConfirm);
// if (!btnBlackConfirmRa.IsEmpty())
// {
// _logger.LogInformation("提交物品:{Text}", "2. 仍旧存在物品");
// continue;
// }
// else
// {
// var btnWhiteConfirmRa = content.CaptureRectArea.Find(ElementAssets.Instance().BtnWhiteConfirm);
// if (!btnWhiteConfirmRa.IsEmpty())
// {
// btnWhiteConfirmRa.ClickCenter();
// _logger.LogInformation("提交物品:{Text}", "3. 交付");
//
// VisionContext.Instance().DrawContent.ClearAll();
// return true;
// }
// break;
// }
// }
// }
// else
// {
// break;
// }
// }
}
return false;
}
}