add feature matcher

This commit is contained in:
辉鸭蛋
2024-04-21 17:18:03 +08:00
parent 25e640bff1
commit f64e2f37c9
9 changed files with 308 additions and 179 deletions

View File

@@ -11,7 +11,7 @@ public class EntireMapTest
{
SpeedTimer speedTimer = new();
var mainMap1024BlockMat = new Mat(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\mainMap1024Block.png", ImreadModes.Grayscale);
var surfMatcher = new SurfMatcher(mainMap1024BlockMat);
var surfMatcher = new FeatureMatcher(mainMap1024BlockMat);
var queryMat = new Mat(@"E:\HuiTask\更好的原神\地图匹配\比较\小地图\Clip_20240323_183119.png", ImreadModes.Grayscale);
speedTimer.Record("初始化特征");

View File

@@ -270,35 +270,6 @@ public class KeyPointMatchTest
return result;
}
[Obsolete("Obsolete")]
public static byte[] SerializeObject(object obj)
{
if (obj == null)
return null;
//内存实例
MemoryStream ms = new MemoryStream();
//创建序列化的实例
BinaryFormatter formatter = new BinaryFormatter();
formatter.Serialize(ms, obj); //序列化对象写入ms流中
byte[] bytes = ms.GetBuffer();
return bytes;
}
[Obsolete("Obsolete")]
public static object DeserializeObject(byte[] bytes)
{
object obj = null;
if (bytes == null)
return obj;
//利用传来的byte[]创建一个内存流
MemoryStream ms = new MemoryStream(bytes);
ms.Position = 0;
BinaryFormatter formatter = new BinaryFormatter();
obj = formatter.Deserialize(ms); //把内存流反序列成对象
ms.Close();
return obj;
}
static bool WriteRawImage(Mat image, string filename)
{
using (FileStream file = new FileStream(filename, FileMode.Create))

View File

@@ -0,0 +1,10 @@
namespace BetterGenshinImpact.Core.Recognition.OpenCv.FeatureMatch;
public enum Feature2DType
{
// ReSharper disable once InconsistentNaming
SURF,
// ReSharper disable once InconsistentNaming
SIFT
}

View File

@@ -5,160 +5,55 @@ using OpenCvSharp.XFeatures2D;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using OpenCvSharp.Features2D;
namespace BetterGenshinImpact.Core.Recognition.OpenCv;
namespace BetterGenshinImpact.Core.Recognition.OpenCv.FeatureMatch;
public class SurfMatcher
public class FeatureMatcher
{
private readonly double _threshold;
private readonly SURF _surf;
private readonly double _threshold = 100; // SURF 100
private readonly Feature2D _feature2D;
private readonly Mat _trainMat; // 大图本身
private readonly Mat _trainRet = new(); // 大图特征描述子
private readonly KeyPoint[] _trainKeyPoints;
private readonly KeyPointFeatureBlock[][] _blocks;
private readonly int _splitRow = 13 * 2;
private readonly int _splitCol = 14 * 2;
private KeyPointFeatureBlock? _lastMergedBlock;
private readonly KeyPointFeatureBlock[][] _blocks; // 特征块存储
private readonly int _splitRow = 13 * 2; // 特征点拆分行数
private readonly int _splitCol = 14 * 2; // 特征点拆分列数
private KeyPointFeatureBlock? _lastMergedBlock; // 上次合并的特征块
public SurfMatcher(Mat trainMat, double threshold = 100)
public FeatureMatcher(Mat trainMat, Feature2DType type = Feature2DType.SIFT, double threshold = 100)
{
_threshold = threshold;
_trainMat = trainMat;
_surf = SURF.Create(_threshold, 4, 3, false, true);
_surf.DetectAndCompute(trainMat, null, out _trainKeyPoints, _trainRet);
if (Feature2DType.SURF == type)
{
_feature2D = SURF.Create(_threshold, 4, 3, false, true);
}
else
{
_feature2D = SIFT.Create();
}
var featureStorage = new FeatureStorage(type, "mainMap1024Block");
var kpFromDisk = featureStorage.LoadKeyPointArray();
if (kpFromDisk == null)
{
_feature2D.DetectAndCompute(trainMat, null, out _trainKeyPoints, _trainRet);
featureStorage.SaveKeyPointArray(_trainKeyPoints);
featureStorage.SaveDescMat(_trainRet);
}
else
{
_trainKeyPoints = kpFromDisk;
_trainRet = featureStorage.LoadDescMat() ?? throw new Exception("加载特征描述矩阵失败");
}
Debug.WriteLine("被匹配的图像生成初始化KeyPoint完成");
_blocks = SplitFeatures(_trainMat, _splitRow, _splitCol, _trainKeyPoints, _trainRet);
_blocks = KeyPointFeatureBlockHelper.SplitFeatures(_trainMat, _splitRow, _splitCol, _trainKeyPoints, _trainRet);
Debug.WriteLine("切割特征点完成");
}
public static KeyPointFeatureBlock[][] SplitFeatures(Mat originalImage, int rows, int cols, KeyPoint[] keyPoints, Mat matches)
{
// Calculate grid size
int cellWidth = originalImage.Width / cols;
int cellHeight = originalImage.Height / rows;
// Initialize arrays to store split features and matches
var splitKeyPoints = new KeyPointFeatureBlock[rows][];
// Initialize each row
for (int i = 0; i < rows; i++)
{
splitKeyPoints[i] = new KeyPointFeatureBlock[cols];
}
// Split features and matches
for (int i = 0; i < keyPoints.Length; i++)
{
int row = (int)(keyPoints[i].Pt.Y / cellHeight);
int col = (int)(keyPoints[i].Pt.X / cellWidth);
// Ensure row and col are within bounds
row = Math.Min(Math.Max(row, 0), rows - 1);
col = Math.Min(Math.Max(col, 0), cols - 1);
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (splitKeyPoints[row][col] == null)
{
splitKeyPoints[row][col] = new KeyPointFeatureBlock();
}
splitKeyPoints[row][col].KeyPointList.Add(keyPoints[i]);
splitKeyPoints[row][col].KeyPointIndexList.Add(i);
}
// 遍历每个特征块,计算描述子
for (int i = 0; i < rows; i++)
{
for (int j = 0; j < cols; j++)
{
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (splitKeyPoints[i][j] == null)
{
continue;
}
var block = splitKeyPoints[i][j];
var descriptor = new Mat(block.KeyPointIndexList.Count, 64, MatType.CV_32FC1);
InitBlockMat(block.KeyPointIndexList, descriptor, matches);
block.Descriptor = descriptor;
}
}
return splitKeyPoints;
}
public static (int, int) GetCellIndex(Mat originalImage, int rows, int cols, int x, int y)
{
// Calculate grid size
int cellWidth = originalImage.Width / cols;
int cellHeight = originalImage.Height / rows;
// Calculate cell index for the given point
int cellRow = (int)(y / cellHeight);
int cellCol = (int)(x / cellWidth);
return (cellRow, cellCol);
}
public static KeyPointFeatureBlock MergeNeighboringFeatures(KeyPointFeatureBlock[][] splitKeyPoints, Mat matches, int cellRow, int cellCol)
{
// Initialize lists to store neighboring features and matches
var neighboringKeyPoints = new List<KeyPoint>();
var neighboringKeyPointIndices = new List<int>();
// Loop through 9 neighboring cells
for (int i = Math.Max(cellRow - 1, 0); i <= Math.Min(cellRow + 1, splitKeyPoints.Length - 1); i++)
{
for (int j = Math.Max(cellCol - 1, 0); j <= Math.Min(cellCol + 1, splitKeyPoints[i].Length - 1); j++)
{
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (splitKeyPoints[i][j] != null)
{
neighboringKeyPoints.AddRange(splitKeyPoints[i][j].KeyPointList);
neighboringKeyPointIndices.AddRange(splitKeyPoints[i][j].KeyPointIndexList);
}
}
}
// Merge neighboring features
var mergedKeyPointBlock = new KeyPointFeatureBlock
{
MergedCenterCellCol = cellCol,
MergedCenterCellRow = cellRow,
KeyPointList = neighboringKeyPoints,
KeyPointIndexList = neighboringKeyPointIndices
};
mergedKeyPointBlock.Descriptor = new Mat(mergedKeyPointBlock.KeyPointIndexList.Count, 64, MatType.CV_32FC1);
InitBlockMat(mergedKeyPointBlock.KeyPointIndexList, mergedKeyPointBlock.Descriptor, matches);
return mergedKeyPointBlock;
}
/// <summary>
/// 按行拷贝matches到descriptor
/// </summary>
/// <param name="keyPointIndexList">记录了哪些行需要拷贝</param>
/// <param name="descriptor">拷贝结果</param>
/// <param name="matches">原始数据</param>
private static unsafe void InitBlockMat(IReadOnlyList<int> keyPointIndexList, Mat descriptor, Mat matches)
{
int cols = matches.Cols;
float* ptrDest = (float*)descriptor.DataPointer;
for (int i = 0; i < keyPointIndexList.Count; i++)
{
var index = keyPointIndexList[i];
float* ptrSrcRow = (float*)matches.Ptr(index);
for (int j = 0; j < cols; j++)
{
*(ptrDest + i * cols + j) = ptrSrcRow[j];
}
}
}
/// <summary>
/// 普通匹配(全图特征)
/// </summary>
@@ -178,11 +73,12 @@ public class SurfMatcher
/// <returns></returns>
public Point2f[]? Match(Mat queryMat, int prevX, int prevY)
{
var (cellRow, cellCol) = GetCellIndex(_trainMat, _splitRow, _splitCol, prevX, prevY);
var (cellRow, cellCol) = KeyPointFeatureBlockHelper.GetCellIndex(_trainMat, _splitRow, _splitCol, prevX, prevY);
if (_lastMergedBlock == null || _lastMergedBlock.MergedCenterCellRow != cellRow || _lastMergedBlock.MergedCenterCellCol != cellCol)
{
_lastMergedBlock = MergeNeighboringFeatures(_blocks, _trainRet, cellRow, cellCol);
_lastMergedBlock = KeyPointFeatureBlockHelper.MergeNeighboringFeatures(_blocks, _trainRet, cellRow, cellCol);
}
return Match(_lastMergedBlock.KeyPointArray, _lastMergedBlock.Descriptor!, queryMat);
}
@@ -199,7 +95,7 @@ public class SurfMatcher
using var queryRet = new Mat();
_surf.DetectAndCompute(queryMat, null, out var queryKeyPoints, queryRet);
_feature2D.DetectAndCompute(queryMat, null, out var queryKeyPoints, queryRet);
speedTimer.Record("模板生成KeyPoint");
using var flnMatcher = new FlannBasedMatcher();

View File

@@ -0,0 +1,82 @@
using BetterGenshinImpact.Core.Config;
using BetterGenshinImpact.Helpers;
using OpenCvSharp;
using System;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
namespace BetterGenshinImpact.Core.Recognition.OpenCv.FeatureMatch;
public class FeatureStorage(Feature2DType type, string name)
{
private readonly string rootPath = Global.Absolute(@"User\Map\");
public KeyPoint[]? LoadKeyPointArray()
{
CreateFolder();
string kpPath = Path.Combine(rootPath, $"{name}_{type}.kp");
if (File.Exists(kpPath))
{
return ObjectUtils.Deserialize(File.ReadAllBytes(kpPath)) as KeyPoint[];
}
return null;
}
public void SaveKeyPointArray(KeyPoint[] kpArray)
{
CreateFolder();
string kpPath = Path.Combine(rootPath, $"{name}_{type}.kp");
File.WriteAllBytes(kpPath, ObjectUtils.Serialize(kpArray));
}
private void CreateFolder()
{
if (Directory.Exists(rootPath) == false)
{
Directory.CreateDirectory(rootPath);
}
}
public Mat? LoadDescMat()
{
CreateFolder();
// 格式: Surf_336767x128.mat
var files = Directory.GetFiles(rootPath, $"{name}_{type}_*.mat", SearchOption.AllDirectories);
if (files.Length == 0)
{
return null;
}
else if (files.Length > 1)
{
Debug.WriteLine($"[FeatureSerializer] Found multiple files: {string.Join(", ", files)}");
}
var rowColPair = Path.GetFileNameWithoutExtension(files[0])
.Replace($"{name}_{type}_", "")
.Split('x');
if (rowColPair.Length != 2)
{
Debug.WriteLine($"[FeatureSerializer] Invalid file name: {files[0]}");
return null;
}
GCHandle pinnedArray = GCHandle.Alloc(ObjectUtils.Deserialize(File.ReadAllBytes(files[0])), GCHandleType.Pinned);
IntPtr pointer = pinnedArray.AddrOfPinnedObject();
return new Mat(Convert.ToInt32(rowColPair[0]), Convert.ToInt32(rowColPair[1]), MatType.CV_32FC1, pointer);
}
public void SaveDescMat(Mat descMat)
{
CreateFolder();
// 删除旧文件
var files = Directory.GetFiles(rootPath, $"{name}_{type}_*.mat", SearchOption.AllDirectories);
foreach (var file in files)
{
File.Delete(file);
}
var descPath = Path.Combine(rootPath, $"{name}_{type}_{descMat.Rows}x{descMat.Cols}.mat");
var bytes = new byte[descMat.Step(0) * descMat.Rows]; // matSrcRet.Total() * matSrcRet.ElemSize()
Marshal.Copy(descMat.Data, bytes, 0, bytes.Length);
File.WriteAllBytes(descPath, ObjectUtils.Serialize(bytes));
}
}

View File

@@ -0,0 +1,138 @@
using BetterGenshinImpact.Core.Recognition.OpenCv.Model;
using OpenCvSharp;
using System;
using System.Collections.Generic;
namespace BetterGenshinImpact.Core.Recognition.OpenCv.FeatureMatch;
public class KeyPointFeatureBlockHelper
{
public static KeyPointFeatureBlock[][] SplitFeatures(Mat originalImage, int rows, int cols, KeyPoint[] keyPoints, Mat matches)
{
var matchesCols = matches.Cols; // SURF 64 SIFT 128
// Calculate grid size
int cellWidth = originalImage.Width / cols;
int cellHeight = originalImage.Height / rows;
// Initialize arrays to store split features and matches
var splitKeyPoints = new KeyPointFeatureBlock[rows][];
// Initialize each row
for (int i = 0; i < rows; i++)
{
splitKeyPoints[i] = new KeyPointFeatureBlock[cols];
}
// Split features and matches
for (int i = 0; i < keyPoints.Length; i++)
{
int row = (int)(keyPoints[i].Pt.Y / cellHeight);
int col = (int)(keyPoints[i].Pt.X / cellWidth);
// Ensure row and col are within bounds
row = Math.Min(Math.Max(row, 0), rows - 1);
col = Math.Min(Math.Max(col, 0), cols - 1);
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (splitKeyPoints[row][col] == null)
{
splitKeyPoints[row][col] = new KeyPointFeatureBlock();
}
splitKeyPoints[row][col].KeyPointList.Add(keyPoints[i]);
splitKeyPoints[row][col].KeyPointIndexList.Add(i);
}
// 遍历每个特征块,计算描述子
for (int i = 0; i < rows; i++)
{
for (int j = 0; j < cols; j++)
{
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (splitKeyPoints[i][j] == null)
{
continue;
}
var block = splitKeyPoints[i][j];
var descriptor = new Mat(block.KeyPointIndexList.Count, matchesCols, MatType.CV_32FC1);
InitBlockMat(block.KeyPointIndexList, descriptor, matches);
block.Descriptor = descriptor;
}
}
return splitKeyPoints;
}
public static (int, int) GetCellIndex(Mat originalImage, int rows, int cols, int x, int y)
{
// Calculate grid size
int cellWidth = originalImage.Width / cols;
int cellHeight = originalImage.Height / rows;
// Calculate cell index for the given point
int cellRow = y / cellHeight;
int cellCol = x / cellWidth;
return (cellRow, cellCol);
}
public static KeyPointFeatureBlock MergeNeighboringFeatures(KeyPointFeatureBlock[][] splitKeyPoints, Mat matches, int cellRow, int cellCol)
{
var matchesCols = matches.Cols; // SURF 64 SIFT 128
// Initialize lists to store neighboring features and matches
var neighboringKeyPoints = new List<KeyPoint>();
var neighboringKeyPointIndices = new List<int>();
// Loop through 9 neighboring cells
for (int i = Math.Max(cellRow - 1, 0); i <= Math.Min(cellRow + 1, splitKeyPoints.Length - 1); i++)
{
for (int j = Math.Max(cellCol - 1, 0); j <= Math.Min(cellCol + 1, splitKeyPoints[i].Length - 1); j++)
{
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (splitKeyPoints[i][j] != null)
{
neighboringKeyPoints.AddRange(splitKeyPoints[i][j].KeyPointList);
neighboringKeyPointIndices.AddRange(splitKeyPoints[i][j].KeyPointIndexList);
}
}
}
// Merge neighboring features
var mergedKeyPointBlock = new KeyPointFeatureBlock
{
MergedCenterCellCol = cellCol,
MergedCenterCellRow = cellRow,
KeyPointList = neighboringKeyPoints,
KeyPointIndexList = neighboringKeyPointIndices
};
mergedKeyPointBlock.Descriptor = new Mat(mergedKeyPointBlock.KeyPointIndexList.Count, matchesCols, MatType.CV_32FC1);
InitBlockMat(mergedKeyPointBlock.KeyPointIndexList, mergedKeyPointBlock.Descriptor, matches);
return mergedKeyPointBlock;
}
/// <summary>
/// 按行拷贝matches到descriptor
/// </summary>
/// <param name="keyPointIndexList">记录了哪些行需要拷贝</param>
/// <param name="descriptor">拷贝结果</param>
/// <param name="matches">原始数据</param>
private static unsafe void InitBlockMat(IReadOnlyList<int> keyPointIndexList, Mat descriptor, Mat matches)
{
int cols = matches.Cols;
float* ptrDest = (float*)descriptor.DataPointer;
for (int i = 0; i < keyPointIndexList.Count; i++)
{
var index = keyPointIndexList[i];
float* ptrSrcRow = (float*)matches.Ptr(index);
for (int j = 0; j < cols; j++)
{
*(ptrDest + i * cols + j) = ptrSrcRow[j];
}
}
}
}

View File

@@ -1,5 +1,6 @@
using BetterGenshinImpact.Core.Config;
using BetterGenshinImpact.Core.Recognition.OpenCv;
using BetterGenshinImpact.Core.Recognition.OpenCv.FeatureMatch;
using CommunityToolkit.Mvvm.Messaging;
using CommunityToolkit.Mvvm.Messaging.Messages;
using OpenCvSharp;
@@ -33,7 +34,7 @@ public class EntireMap
/// </summary>
private readonly Mat _cityMap2048BlockMat;
private readonly SurfMatcher _surfMatcher;
private readonly FeatureMatcher _surfMatcher;
private int _prevX = -1;
private int _prevY = -1;
@@ -44,7 +45,7 @@ public class EntireMap
_mainMap100BlockMat = new Mat(Global.Absolute(@"Assets\Map\mainMap100Block.png"));
_mainMap1024BlockMat = new Mat(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\mainMap1024Block.png", ImreadModes.Grayscale);
// _cityMap2048BlockMat = new Mat(@"E:\HuiTask\更好的原神\地图匹配\有用的素材\cityMap2048Block.png", ImreadModes.Grayscale);
_surfMatcher = new SurfMatcher(_mainMap1024BlockMat);
_surfMatcher = new FeatureMatcher(_mainMap1024BlockMat);
}
/// <summary>
@@ -89,8 +90,7 @@ public class EntireMap
if (pArray == null || pArray.Length < 4)
{
_prevX = -1;
_prevY = -1;
(_prevX, _prevY) = (-1, -1);
throw new InvalidOperationException();
}
var rect = Cv2.BoundingRect(pArray);
@@ -128,6 +128,7 @@ public class EntireMap
}
catch (Exception)
{
(_prevX, _prevY) = (-1, -1);
Debug.WriteLine("Surf Match Failed");
}
}

View File

@@ -6,15 +6,13 @@ using BetterGenshinImpact.Helpers.Extensions;
using BetterGenshinImpact.View.Drawable;
using OpenCvSharp;
using OpenCvSharp.Extensions;
using Sdcb.PaddleOCR;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using Sdcb.PaddleOCR;
using Point = OpenCvSharp.Point;
using static BetterGenshinImpact.GameTask.Common.TaskControl;
namespace BetterGenshinImpact.GameTask.Model;

View File

@@ -0,0 +1,33 @@
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
namespace BetterGenshinImpact.Helpers;
public class ObjectUtils
{
public static byte[] Serialize(object obj)
{
var ms = new MemoryStream();
var formatter = new BinaryFormatter();
#pragma warning disable SYSLIB0011
formatter.Serialize(ms, obj);
#pragma warning restore SYSLIB0011
byte[] bytes = ms.GetBuffer();
return bytes;
}
public static object Deserialize(byte[] bytes)
{
//利用传来的byte[]创建一个内存流
var ms = new MemoryStream(bytes)
{
Position = 0
};
var formatter = new BinaryFormatter();
#pragma warning disable SYSLIB0011
var obj = formatter.Deserialize(ms); //把内存流反序列成对象
#pragma warning restore SYSLIB0011
ms.Close();
return obj;
}
}