feat: finsh auto skip

This commit is contained in:
huiyadanli
2023-09-24 00:52:28 +08:00
parent 6edb3825ef
commit b5d9ce1c93
19 changed files with 335 additions and 173 deletions

View File

@@ -34,6 +34,9 @@
</ItemGroup>
<ItemGroup>
<None Update="GameTask\AutoSkip\Assets\1920x1080\menu.png">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="GameTask\AutoSkip\Assets\1920x1080\option.png">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 386 B

After

Width:  |  Height:  |  Size: 480 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 313 B

After

Width:  |  Height:  |  Size: 319 B

View File

@@ -10,7 +10,8 @@ namespace BetterGenshinImpact.GameTask.AutoSkip.Assets
{
public class AutoSkipAssets
{
public static Mat StopAutoButtonMat = new(Global.Absolute("GameTask/AutoSkip/Assets/stop_auto.png"), ImreadModes.Grayscale);
public static Mat OptionMat = new(Global.Absolute("GameTask/AutoSkip/Assets/option.png"), ImreadModes.Grayscale);
public static Mat StopAutoButtonMat = new(Global.Absolute(@"GameTask\AutoSkip\Assets\1920x1080\stop_auto.png"), ImreadModes.Grayscale);
public static Mat OptionMat = new(Global.Absolute(@"GameTask\AutoSkip\Assets\1920x1080\option.png"), ImreadModes.Grayscale);
public static Mat MenuMat = new(Global.Absolute(@"GameTask\AutoSkip\Assets\1920x1080\menu.png"), ImreadModes.Grayscale);
}
}

View File

@@ -1,5 +1,8 @@
using System;
using System.Diagnostics;
using BetterGenshinImpact.GameTask.AutoSkip.Assets;
using BetterGenshinImpact.Utils.Extensions;
using Microsoft.Extensions.Logging;
using OpenCvSharp;
using Vision.Recognition.Helper.OpenCv;
using Vision.Recognition.Task;
@@ -9,44 +12,68 @@ namespace BetterGenshinImpact.GameTask.AutoSkip
{
public class AutoSkipTrigger : ITaskTrigger
{
private ILogger<AutoSkipTrigger> _logger = App.GetLogger<AutoSkipTrigger>();
public string Name => "自动剧情";
public bool IsEnabled { get; set; }
public int Priority => 20;
public bool IsExclusive => false;
public void Init(ITaskContext context)
public void Init()
{
IsEnabled = true;
}
public void OnCapture(Mat matSrc, int frameIndex)
{
//TODO 切割图片加快效率
if (frameIndex % 2 == 0)
{
return;
}
var grayMat = new Mat();
Cv2.CvtColor(matSrc, grayMat, ColorConversionCodes.BGR2GRAY);
// 找左上角剧情自动的按钮
var p1 = MatchTemplateHelper.FindSingleTarget(grayMat, AutoSkipAssets.StopAutoButtonMat);
var grayLeftTopMat = CutHelper.CutLeftTop(grayMat, grayMat.Width / 5, grayMat.Height / 5);
var p1 = MatchTemplateHelper.FindSingleTarget(grayLeftTopMat, AutoSkipAssets.StopAutoButtonMat, 0.9);
if (p1 is { X: > 0, Y: > 0 })
{
//TODO 无效操作代码 需要替换
new InputSimulator().Keyboard.KeyPress(VirtualKeyCode.SPACE);
return;
Debug.WriteLine($"按下空格");
}
// 不存在则找右下的选项按钮
var p2 = MatchTemplateHelper.FindSingleTarget(grayMat, AutoSkipAssets.OptionMat);
var grayRightBottomMat = CutHelper.CutRightBottom(grayMat, grayMat.Width / 2, grayMat.Height / 3 * 2);
var p2 = MatchTemplateHelper.FindSingleTarget(grayRightBottomMat, AutoSkipAssets.OptionMat);
if (p2 is { X: > 0, Y: > 0 })
{
new InputSimulator().Mouse.MoveMouseTo(p2.X, p2.Y).LeftButtonClick();
return;
// 不存在菜单的情况下 剧情在播放中
var grayLeftTopMat2 = CutHelper.CutLeftTop(grayMat, grayMat.Width / 4, grayMat.Height / 4);
var pMenu = MatchTemplateHelper.FindSingleTarget(grayLeftTopMat2, AutoSkipAssets.MenuMat);
if (pMenu is { X: 0, Y: 0 })
{
p2 = p2.ToDesktopPositionOffset65535(grayMat.Width - grayMat.Width / 2,
grayMat.Height - grayMat.Height / 3 * 2);
new InputSimulator().Mouse.MoveMouseTo(p2.X, p2.Y).LeftButtonClick();
_logger.LogInformation($"点击选项按钮:{p2}");
Debug.WriteLine($"点击选项按钮:{p2}");
return;
}
}
// 判断左上角的黑色像素个数
// 黑屏剧情要点击鼠标(多次) 几乎全黑的时候不用点击
var blackCount = OpenCvCommonHelper.CountGrayMatColor(grayMat, 0);
var rate = blackCount * 1.0 / (grayMat.Width * grayMat.Height);
if (rate > 0.9)
if (rate > 0.7 && rate < 0.99)
{
//TODO click center
var p3 = new Point(grayMat.Width / 2, grayMat.Height / 2).ToDesktopPosition65535();
new InputSimulator().Mouse.MoveMouseTo(p3.X, p3.Y).LeftButtonClick();
Debug.WriteLine($"点击黑屏剧情:{rate}");
return;
}
// TODO 自动交付材料
}
}
}

View File

@@ -33,6 +33,8 @@ namespace BetterGenshinImpact.GameTask
List<ITaskTrigger> loadedTriggers = new();
loadedTriggers.Add(new AutoSkip.AutoSkipTrigger());
loadedTriggers.ForEach(i => i.Init());
return loadedTriggers.OrderByDescending(i => i.Priority).ToList();
}

View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
using Vanara.PInvoke;
@@ -50,5 +51,32 @@ namespace BetterGenshinImpact.GameTask
return null;
}
}
/// <summary>
/// 获取窗口位置
/// </summary>
/// <param name="hWnd"></param>
/// <returns></returns>
public static RECT GetWindowRect(IntPtr hWnd)
{
User32.GetWindowRect(hWnd, out var windowRect);
return windowRect;
}
/// <summary>
/// 游戏本身分辨率获取
/// </summary>
/// <param name="hWnd"></param>
/// <returns></returns>
public static RECT GetGameScreenRect(IntPtr hWnd)
{
User32.GetClientRect(hWnd, out var clientRect);
return clientRect;
}
//public static int GetCaptionHeight()
//{
// return User32.GetSystemMetrics(User32.SystemMetric.SM_CYFRAME) + User32.GetSystemMetrics(User32.SystemMetric.SM_CYCAPTION);
//}
}
}

View File

@@ -0,0 +1,37 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Vision.Recognition;
namespace BetterGenshinImpact.GameTask
{
/// <summary>
/// 任务上下文
/// </summary>
public class TaskContext
{
private static TaskContext? _uniqueInstance;
private static readonly object Locker = new();
private TaskContext()
{
}
public static TaskContext Instance()
{
if (_uniqueInstance == null)
{
lock (Locker)
{
_uniqueInstance ??= new TaskContext();
}
}
return _uniqueInstance;
}
public IntPtr GameHandle { get; set; }
}
}

View File

@@ -20,13 +20,13 @@ namespace BetterGenshinImpact.GameTask
private readonly ILogger<TaskDispatcher> _logger = App.GetLogger<TaskDispatcher>();
private readonly Timer _timer = new();
private List<ITaskTrigger> _triggers = new();
private readonly List<ITaskTrigger> _triggers;
private IWindowCapture? _capture;
private int _frameIndex = 0;
private int _frameRate = 60;
private int _frameRate = 30;
public TaskDispatcher()
@@ -36,7 +36,7 @@ namespace BetterGenshinImpact.GameTask
_timer.Elapsed += Tick;
}
public void Start(CaptureMode mode, int frameRate)
public void Start(CaptureMode mode, int frameRate = 30)
{
IntPtr hWnd = SystemControl.FindGenshinImpactHandle();
if (hWnd == IntPtr.Zero)
@@ -44,6 +44,7 @@ namespace BetterGenshinImpact.GameTask
MessageBox.Show("未找到原神窗口");
return;
}
TaskContext.Instance().GameHandle = hWnd;
_frameRate = frameRate;
@@ -63,22 +64,29 @@ namespace BetterGenshinImpact.GameTask
public void Tick(object? sender, EventArgs e)
{
if (_capture == null)
// 检查截图器是否初始化
if (_capture == null || !_capture.IsCapturing)
{
_logger.LogError("截图器未初始化!");
Stop();
return;
}
// 检查游戏是否在前台
if (!SystemControl.IsGenshinImpactActive())
{
return;
}
// 帧序号自增 1分钟后归零
_frameIndex = (_frameIndex + 1) % (_frameRate * 60);
// 捕获游戏画面
var sw = new Stopwatch();
sw.Start();
//var sw = new Stopwatch();
//sw.Start();
var bitmap = _capture.Capture();
sw.Stop();
Debug.WriteLine("截图耗时:" + sw.ElapsedMilliseconds);
//sw.Stop();
//Debug.WriteLine("截图耗时:" + sw.ElapsedMilliseconds);
if (bitmap == null)
{

View File

@@ -0,0 +1,48 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BetterGenshinImpact.GameTask;
using OpenCvSharp;
using Vision.Recognition.Helper.Simulator;
namespace BetterGenshinImpact.Utils.Extensions
{
public static class PointExtension
{
public static Point ToDesktopPosition(this Point point)
{
if (TaskContext.Instance().GameHandle == IntPtr.Zero)
{
return point;
}
var rc = SystemControl.GetWindowRect(TaskContext.Instance().GameHandle);
return new Point(rc.X + point.X, rc.Y + point.Y);
}
public static Point ToDesktopPosition65535(this Point point)
{
var p = point.ToDesktopPosition();
return new Point(p.X * 65535 / PrimaryScreen.WorkingArea.Width, p.Y * 65535 / PrimaryScreen.WorkingArea.Height);
}
public static Point ToDesktopPositionOffset(this Point point, int offsetX, int offsetY)
{
if (TaskContext.Instance().GameHandle == IntPtr.Zero)
{
return point;
}
var rc = SystemControl.GetWindowRect(TaskContext.Instance().GameHandle);
return new Point(rc.X + point.X + offsetX, rc.Y + point.Y + offsetY);
}
public static Point ToDesktopPositionOffset65535(this Point point, int offsetX, int offsetY)
{
var p = point.ToDesktopPositionOffset(offsetX, offsetY);
return new Point(p.X * 65535 / PrimaryScreen.WorkingArea.Width, p.Y * 65535 / PrimaryScreen.WorkingArea.Height);
}
}
}

View File

@@ -1,99 +0,0 @@
using System;
using System.Drawing;
using System.Runtime.InteropServices;
using Vanara.PInvoke;
using static Vanara.PInvoke.Gdi32;
namespace BetterGenshinImpact.Utils
{
public class PrimaryScreen
{
/// <summary>
/// 获取屏幕分辨率当前物理大小
/// </summary>
public static Size WorkingArea
{
get
{
var hdc = User32.GetDC(IntPtr.Zero);
var size = new Size
{
Width = Gdi32.GetDeviceCaps(hdc, DeviceCap.HORZRES),
Height = Gdi32.GetDeviceCaps(hdc, DeviceCap.VERTRES)
};
User32.ReleaseDC(IntPtr.Zero, hdc);
return size;
}
}
/// <summary>
/// 当前系统DPI_X 大小 一般为96
/// </summary>
public static int DpiX
{
get
{
var hdc = User32.GetDC(IntPtr.Zero);
var dpiX = Gdi32.GetDeviceCaps(hdc, DeviceCap.LOGPIXELSX);
User32.ReleaseDC(IntPtr.Zero, hdc);
return dpiX;
}
}
/// <summary>
/// 当前系统DPI_Y 大小 一般为96
/// </summary>
public static int DpiY
{
get
{
var hdc = User32.GetDC(IntPtr.Zero);
var dpiX = Gdi32.GetDeviceCaps(hdc, DeviceCap.LOGPIXELSY);
User32.ReleaseDC(IntPtr.Zero, hdc);
return dpiX;
}
}
/// <summary>
/// 获取真实设置的桌面分辨率大小
/// </summary>
public static Size DESKTOP
{
get
{
var hdc = User32.GetDC(IntPtr.Zero);
var size = new Size
{
Width = Gdi32.GetDeviceCaps(hdc, DeviceCap.DESKTOPHORZRES),
Height = Gdi32.GetDeviceCaps(hdc, DeviceCap.DESKTOPVERTRES)
};
User32.ReleaseDC(IntPtr.Zero, hdc);
return size;
}
}
/// <summary>
/// 获取宽度缩放百分比
/// </summary>
public static float ScaleX
{
get
{
var hdc = User32.GetDC(IntPtr.Zero);
var scaleX = (float)Gdi32.GetDeviceCaps(hdc, DeviceCap.DESKTOPHORZRES) / (float)Gdi32.GetDeviceCaps(hdc, DeviceCap.HORZRES);
User32.ReleaseDC(IntPtr.Zero, hdc);
return scaleX;
}
}
/// <summary>
/// 获取高度缩放百分比
/// </summary>
public static float ScaleY
{
get
{
var hdc = User32.GetDC(IntPtr.Zero);
var scaleY = (float)Gdi32.GetDeviceCaps(hdc, DeviceCap.DESKTOPVERTRES) / (float)Gdi32.GetDeviceCaps(hdc, DeviceCap.VERTRES);
User32.ReleaseDC(IntPtr.Zero, hdc);
return scaleY;
}
}
}
}

View File

@@ -26,5 +26,6 @@
<ComboBox x:Name="CboCaptureType" HorizontalAlignment="Left" Height="23" Margin="10,20,0,0"
VerticalAlignment="Top" Width="78" ItemsSource="{Binding ModeNames}"
SelectedItem="{Binding SelectedMode, Mode=TwoWay}" />
<Button x:Name="Start" Content="Start" Command="{Binding StartCaptureCommand}" HorizontalAlignment="Left" Margin="10,63,0,0" VerticalAlignment="Top" Height="33" Width="78" />
</Grid>
</Window>

View File

@@ -1,13 +1,19 @@
using CommunityToolkit.Mvvm.ComponentModel;
using BetterGenshinImpact.GameTask;
using CommunityToolkit.Mvvm.ComponentModel;
using CommunityToolkit.Mvvm.Input;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using BetterGenshinImpact.Core.Config;
using OpenCvSharp;
using Vanara.PInvoke;
using Vision.Recognition;
using Vision.Recognition.Helper.Simulator;
using Vision.WindowCapture;
namespace BetterGenshinImpact.ViewModel
@@ -25,7 +31,8 @@ namespace BetterGenshinImpact.ViewModel
[RelayCommand]
private void OnLoaded()
{
TestMask();
//TestMask();
//TestRect();
}
[RelayCommand]
@@ -35,21 +42,65 @@ namespace BetterGenshinImpact.ViewModel
Application.Current.Shutdown();
}
private void TestMask()
[RelayCommand]
private void OnStartCapture()
{
//var hWnd = FindGenshinImpactHandle();
//if (hWnd == IntPtr.Zero)
//{
// MessageBox.Show("未找到原神窗口");
// return;
//}
TestMask();
new TaskDispatcher().Start(CaptureMode.BitBlt);
}
//User32.GetWindowRect(hWnd, out var rect);
private void TestRect()
{
var hWnd = SystemControl.FindGenshinImpactHandle();
if (hWnd == IntPtr.Zero)
{
MessageBox.Show("未找到原神窗口");
return;
}
User32.GetWindowRect(hWnd, out var rect);
//var x = rect.X;
//var y = rect.Y;
//var w = rect.Width;
//var h = rect.Height;
var x = (int)Math.Ceiling(rect.X * PrimaryScreen.ScaleX);
var y = (int)Math.Ceiling(rect.Y * PrimaryScreen.ScaleY);
var w = (int)Math.Ceiling(rect.Width * PrimaryScreen.ScaleX);
var h = (int)Math.Ceiling(rect.Height * PrimaryScreen.ScaleY);
Debug.WriteLine($"原神窗口大小:{rect.Width} x {rect.Height}");
Debug.WriteLine($"原神窗口大小(计算DPI缩放后){w} x {h}");
User32.GetClientRect(hWnd, out var clientRect);
var cx = clientRect.X;
var cy = clientRect.Y;
var cw = clientRect.Width;
var ch = clientRect.Height;
Debug.WriteLine($"原神窗口内控件大小:{clientRect.Width} x {clientRect.Height}");
var h2 = User32.GetSystemMetrics(User32.SystemMetric.SM_CYFRAME);
var h3 = User32.GetSystemMetrics(User32.SystemMetric.SM_CYCAPTION);
_logger.LogInformation($"标题栏高度: {h2} {h3}");
}
private void TestMask()
{
var hWnd = SystemControl.FindGenshinImpactHandle();
if (hWnd == IntPtr.Zero)
{
MessageBox.Show("未找到原神窗口");
return;
}
User32.GetWindowRect(hWnd, out var rect);
var x = rect.X;
var y = rect.Y;
var w = rect.Width;
var h = rect.Height;
//var x = (int)Math.Ceiling(rect.X * PrimaryScreen.ScaleX);
//var y = (int)Math.Ceiling(rect.Y * PrimaryScreen.ScaleY);
//var w = (int)Math.Ceiling(rect.Width * PrimaryScreen.ScaleX);
@@ -57,10 +108,10 @@ namespace BetterGenshinImpact.ViewModel
//Debug.WriteLine($"原神窗口大小:{rect.Width} x {rect.Height}");
//Debug.WriteLine($"原神窗口大小(计算DPI缩放后){w} x {h}");
var x = 0;
var y = 0;
var w = 1200;
var h = 800;
//var x = 0;
//var y = 0;
//var w = 1200;
//var h = 800;
_maskWindow = MaskWindow.Instance();
////window.Owner = this;

View File

@@ -0,0 +1,64 @@
using OpenCvSharp;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Vision.Recognition.Helper.OpenCv
{
/// <summary>
/// 图片剪裁
/// </summary>
public class CutHelper
{
public static Mat CutRightTop(Mat srcMat, int saveRightWidth, int saveTopHeight)
{
srcMat = new Mat(srcMat, new Rect(srcMat.Width - saveRightWidth, 0, saveRightWidth, saveTopHeight));
return srcMat;
}
public static Mat CutRightBottom(Mat srcMat, int saveRightWidth, int saveBottomHeight)
{
srcMat = new Mat(srcMat, new Rect(srcMat.Width - saveRightWidth, srcMat.Height - saveBottomHeight, saveRightWidth, saveBottomHeight));
return srcMat;
}
public static Mat CutLeftTop(Mat srcMat, int saveLeftWidth, int saveTopHeight)
{
srcMat = new Mat(srcMat, new Rect(0, 0, saveLeftWidth, saveTopHeight));
return srcMat;
}
public static Mat CutLeftBottom(Mat srcMat, int saveLeftWidth, int saveBottomHeight)
{
srcMat = new Mat(srcMat, new Rect(0, srcMat.Height - saveBottomHeight, saveLeftWidth, saveBottomHeight));
return srcMat;
}
public static Mat CutTop(Mat srcMat, int saveTopHeight)
{
srcMat = new Mat(srcMat, new Rect(0, 0, srcMat.Width, saveTopHeight));
return srcMat;
}
public static Mat CutBottom(Mat srcMat, int saveBottomHeight)
{
srcMat = new Mat(srcMat, new Rect(0, srcMat.Height- saveBottomHeight, srcMat.Width, saveBottomHeight));
return srcMat;
}
public static Mat CutRight(Mat srcMat, int saveRightWidth)
{
srcMat = new Mat(srcMat, new Rect(srcMat.Width - saveRightWidth, 0, saveRightWidth, srcMat.Height));
return srcMat;
}
public static Mat CutLeft(Mat srcMat, int saveLeftWidth)
{
srcMat = new Mat(srcMat, new Rect(0, 0, saveLeftWidth, srcMat.Height));
return srcMat;
}
}
}

View File

@@ -17,13 +17,10 @@ namespace Vision.Recognition.Task
protected CancellationTokenSource Cancellation;
protected ITaskContext TaskContext;
public BaseTaskThread(ILogger<BaseTaskThread> logger, CancellationTokenSource cts, ITaskContext taskContext)
public BaseTaskThread(ILogger<BaseTaskThread> logger, CancellationTokenSource cts)
{
Log = logger;
Cancellation = cts;
TaskContext = taskContext;
}
}
}

View File

@@ -1,18 +0,0 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Vision.Recognition.Task
{
/// <summary>
/// 任务上下文
/// </summary>
public interface ITaskContext
{
Bitmap Capture();
}
}

View File

@@ -41,8 +41,7 @@ namespace Vision.Recognition.Task
/// <summary>
/// 初始化
/// </summary>
/// <param name="context"></param>
void Init(ITaskContext context);
void Init();
/// <summary>
/// 捕获图像后操作

View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.Linq;
using System.Reflection.Metadata;
@@ -16,10 +17,14 @@ namespace Vision.WindowCapture.BitBlt
private IntPtr _hWnd;
public bool IsCapturing { get; private set; }
private HDC hdcSrc;
public void Start(IntPtr hWnd)
{
_hWnd = hWnd;
IsCapturing = true;
hdcSrc = User32.GetWindowDC(_hWnd);
}
public Bitmap? Capture()
@@ -29,22 +34,30 @@ namespace Vision.WindowCapture.BitBlt
return null;
}
User32.GetWindowRect(_hWnd, out var windowRect);
var width = windowRect.right - windowRect.left;
var height = windowRect.bottom - windowRect.top;
try
{
User32.GetWindowRect(_hWnd, out var windowRect);
var width = windowRect.Width;
var height = windowRect.Height;
var hdcSrc = User32.GetWindowDC(_hWnd);
var hdcDest = Gdi32.CreateCompatibleDC(hdcSrc);
var hBitmap = Gdi32.CreateCompatibleBitmap(hdcSrc, width, height);
var hOld = Gdi32.SelectObject(hdcDest, hBitmap);
Gdi32.BitBlt(hdcDest, 0, 0, width, height, hdcSrc, 0, 0, Gdi32.RasterOperationMode.SRCCOPY);
Gdi32.SelectObject(hdcDest, hOld);
Gdi32.DeleteDC(hdcDest);
User32.ReleaseDC(_hWnd, hdcSrc);
var bitmap = hBitmap.ToBitmap();
Gdi32.DeleteObject(hBitmap);
return bitmap;
var hdcDest = Gdi32.CreateCompatibleDC(hdcSrc);
var hBitmap = Gdi32.CreateCompatibleBitmap(hdcSrc, width, height);
var hOld = Gdi32.SelectObject(hdcDest, hBitmap);
Gdi32.BitBlt(hdcDest, 0, 0, width, height, hdcSrc, 0, 0, Gdi32.RasterOperationMode.SRCCOPY);
Gdi32.SelectObject(hdcDest, hOld);
Gdi32.DeleteDC(hdcDest);
User32.ReleaseDC(_hWnd, hdcSrc);
var bitmap = hBitmap.ToBitmap();
Gdi32.DeleteObject(hBitmap);
return bitmap;
}
catch (Exception e)
{
Debug.WriteLine(e);
return null;
}
}
public void Stop()