perf(geartask): 按需导出路径仓库文件以提升初始化性能

- 将路径仓库的镜像初始化从全量导出改为按需导出,避免首次转换时不必要的文件复制
- 添加目录级导出缓存,避免重复导出同一目录下的文件
- 重构 ScriptRepoUpdater 以支持文件/目录的按需导出功能
This commit is contained in:
辉鸭蛋
2026-05-09 04:52:49 +08:00
parent 0f9bd7f465
commit 00e62f80ee
2 changed files with 273 additions and 60 deletions

View File

@@ -1674,6 +1674,101 @@ public class ScriptRepoUpdater : Singleton<ScriptRepoUpdater>
/// <summary>
/// 从Git仓库读取文件内容
/// </summary>
/// <summary>
/// 将中央仓库中的单个文件导出到本地。
/// </summary>
/// <param name="relPath">相对于仓库 `repo/` 目录的路径。</param>
/// <param name="localPath">本地目标文件路径。</param>
/// <returns>是否导出成功。</returns>
public bool ExportFileFromCenterRepo(string relPath, string localPath)
{
try
{
var normalizedRelPath = NormalizeRepoRelativePath(relPath);
var repoPath = CenterRepoPath;
var localDirectory = Path.GetDirectoryName(localPath);
if (!string.IsNullOrEmpty(localDirectory))
{
Directory.CreateDirectory(localDirectory);
}
if (IsGitRepository(repoPath))
{
using var repo = new Repository(repoPath);
if (!TryGetBlobByRelativePath(repo, normalizedRelPath, out var blob))
{
return false;
}
using var contentStream = blob.GetContentStream();
using var fileStream = File.Create(localPath);
contentStream.CopyTo(fileStream);
return true;
}
var sourceFilePath = Path.Combine(repoPath, "repo", normalizedRelPath);
if (!File.Exists(sourceFilePath))
{
return false;
}
File.Copy(sourceFilePath, localPath, true);
return true;
}
catch (Exception ex)
{
_logger.LogError(ex, "导出仓库文件失败: {RelPath}", relPath);
return false;
}
}
/// <summary>
/// 将中央仓库目录下指定扩展名的文件递归导出到本地。
/// </summary>
/// <param name="relDir">相对于仓库 `repo/` 目录的路径。</param>
/// <param name="targetDir">本地目标目录。</param>
/// <param name="extensionWithDot">要导出的文件扩展名。</param>
public void ExportFilesFromCenterRepo(string relDir, string targetDir, string extensionWithDot)
{
try
{
var normalizedRelDir = NormalizeRepoRelativePath(relDir);
var normalizedExtension = extensionWithDot.StartsWith(".")
? extensionWithDot
: "." + extensionWithDot;
var repoPath = CenterRepoPath;
Directory.CreateDirectory(targetDir);
if (IsGitRepository(repoPath))
{
using var repo = new Repository(repoPath);
var rootTree = GetRepoSubdirectoryTree(repo);
if (!TryGetTreeByRelativePath(rootTree, normalizedRelDir, out var targetTree))
{
return;
}
ExportFilesFromGitTree(targetTree, targetDir, normalizedExtension);
return;
}
var sourceDir = string.IsNullOrEmpty(normalizedRelDir)
? Path.Combine(repoPath, "repo")
: Path.Combine(repoPath, "repo", normalizedRelDir);
if (!Directory.Exists(sourceDir))
{
return;
}
ExportFilesFromDirectory(sourceDir, targetDir, normalizedExtension);
}
catch (Exception ex)
{
_logger.LogError(ex, "导出仓库目录失败: {RelDir}", relDir);
}
}
private string? ReadFileFromGitRepository(string repoPath, string filePath)
{
try
@@ -1687,35 +1782,11 @@ public class ScriptRepoUpdater : Singleton<ScriptRepoUpdater>
using var repo = new Repository(repoPath);
var manifestPath = $"repo/{filePath}";
var pathParts = manifestPath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
Tree currentTree = repo.Head.Tip!.Tree;
TreeEntry? entry = null;
for (int i = 0; i < pathParts.Length; i++)
{
entry = currentTree[pathParts[i]];
if (entry == null)
{
return null;
}
if (i < pathParts.Length - 1)
{
if (entry.TargetType != TreeEntryTargetType.Tree)
{
return null;
}
currentTree = (Tree)entry.Target;
}
}
if (entry == null || entry.TargetType != TreeEntryTargetType.Blob)
if (!TryGetBlobByRelativePath(repo, filePath, out var blob))
{
return null;
}
var blob = (Blob)entry.Target;
using var contentStream = blob.GetContentStream();
using var reader = new StreamReader(contentStream);
return reader.ReadToEnd();
@@ -1730,6 +1801,87 @@ public class ScriptRepoUpdater : Singleton<ScriptRepoUpdater>
/// <summary>
/// 从Git仓库读取二进制文件内容
/// </summary>
private static bool TryGetBlobByRelativePath(Repository repo, string relPath, out Blob blob)
{
blob = null!;
var manifestPath = $"repo/{NormalizeRepoRelativePath(relPath)}";
var pathParts = manifestPath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
Tree currentTree = repo.Head.Tip!.Tree;
TreeEntry? entry = null;
for (int i = 0; i < pathParts.Length; i++)
{
entry = currentTree[pathParts[i]];
if (entry == null)
{
return false;
}
if (i < pathParts.Length - 1)
{
if (entry.TargetType != TreeEntryTargetType.Tree)
{
return false;
}
currentTree = (Tree)entry.Target;
}
}
if (entry == null || entry.TargetType != TreeEntryTargetType.Blob)
{
return false;
}
blob = (Blob)entry.Target;
return true;
}
private static void ExportFilesFromGitTree(Tree tree, string targetDir, string extensionWithDot)
{
foreach (var entry in tree)
{
switch (entry.TargetType)
{
case TreeEntryTargetType.Tree:
var childTargetDir = Path.Combine(targetDir, entry.Name);
Directory.CreateDirectory(childTargetDir);
ExportFilesFromGitTree((Tree)entry.Target, childTargetDir, extensionWithDot);
break;
case TreeEntryTargetType.Blob when entry.Name.EndsWith(extensionWithDot, StringComparison.OrdinalIgnoreCase):
var filePath = Path.Combine(targetDir, entry.Name);
using (var contentStream = ((Blob)entry.Target).GetContentStream())
using (var fileStream = File.Create(filePath))
{
contentStream.CopyTo(fileStream);
}
break;
}
}
}
private static void ExportFilesFromDirectory(string sourceDir, string targetDir, string extensionWithDot)
{
foreach (var directory in Directory.GetDirectories(sourceDir))
{
var directoryName = Path.GetFileName(directory);
var childTargetDir = Path.Combine(targetDir, directoryName);
Directory.CreateDirectory(childTargetDir);
ExportFilesFromDirectory(directory, childTargetDir, extensionWithDot);
}
foreach (var file in Directory.GetFiles(sourceDir))
{
if (!file.EndsWith(extensionWithDot, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var targetFilePath = Path.Combine(targetDir, Path.GetFileName(file));
File.Copy(file, targetFilePath, true);
}
}
private byte[]? ReadBinaryFileFromGitRepository(string repoPath, string filePath)
{
try
@@ -1743,35 +1895,11 @@ public class ScriptRepoUpdater : Singleton<ScriptRepoUpdater>
using var repo = new Repository(repoPath);
var manifestPath = $"repo/{filePath}";
var pathParts = manifestPath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
Tree currentTree = repo.Head.Tip!.Tree;
TreeEntry? entry = null;
for (int i = 0; i < pathParts.Length; i++)
{
entry = currentTree[pathParts[i]];
if (entry == null)
{
return null;
}
if (i < pathParts.Length - 1)
{
if (entry.TargetType != TreeEntryTargetType.Tree)
{
return null;
}
currentTree = (Tree)entry.Target;
}
}
if (entry == null || entry.TargetType != TreeEntryTargetType.Blob)
if (!TryGetBlobByRelativePath(repo, filePath, out var blob))
{
return null;
}
var blob = (Blob)entry.Target;
using var contentStream = blob.GetContentStream();
using var memoryStream = new MemoryStream();
contentStream.CopyTo(memoryStream);

View File

@@ -26,6 +26,7 @@ public class GearTaskConverter
private readonly GearTaskFactory _taskFactory;
private readonly object _mirrorLock = new();
private bool _pathingRepoMirrorInitialized;
private readonly HashSet<string> _exportedPathingRepoDirectories = new(StringComparer.OrdinalIgnoreCase);
public GearTaskConverter(ILogger<GearTaskConverter> logger, GearTaskFactory taskFactory)
{
@@ -327,6 +328,7 @@ public class GearTaskConverter
private List<GearTaskData> BuildPathingReferenceChildren(string repoRelativePath)
{
var result = new List<GearTaskData>();
EnsurePathingRepoDirectoryExported(repoRelativePath);
var children = ScriptRepoUpdater.Instance.GetChildrenFromCenterRepo(repoRelativePath);
foreach (var entry in children)
{
@@ -349,8 +351,10 @@ public class GearTaskConverter
continue;
}
var executionPath = GetPathingExecutionFilePath(entry.RelativePath);
var parameters = new PathingGearTaskParams { Path = executionPath };
var parameters = new PathingGearTaskParams
{
Path = BuildPathingPlaceholderPath(entry.RelativePath, false),
};
result.Add(new GearTaskData
{
Name = Path.GetFileNameWithoutExtension(entry.Name),
@@ -373,7 +377,8 @@ public class GearTaskConverter
}
var parameters = DeserializePathingParams(taskData.Parameters);
if (!string.IsNullOrWhiteSpace(parameters.Path))
if (!string.IsNullOrWhiteSpace(parameters.Path)
&& !TryExtractPathingRepoRelativePath(parameters.Path, out _))
{
return taskData;
}
@@ -473,20 +478,36 @@ public class GearTaskConverter
private string GetPathingExecutionFilePath(string repoRelativeJsonPath)
{
EnsurePathingRepoMirrorInitialized();
// Pathing 文件改为按需导出,避免首次转换时全量镜像仓库
var normalized = repoRelativeJsonPath.Replace('\\', '/').Trim('/');
var relativeUnderPathing = normalized.StartsWith("pathing/", StringComparison.OrdinalIgnoreCase)
? normalized["pathing/".Length..]
: normalized;
var mirrorRoot = GetPathingRepoMirrorRoot();
var target = Path.Combine(mirrorRoot, relativeUnderPathing.Replace('/', Path.DirectorySeparatorChar));
var target = GetPathingMirrorPath(normalized);
if (File.Exists(target))
{
return target;
}
lock (_mirrorLock)
{
if (File.Exists(target))
{
return target;
}
var exportDirectory = Path.GetDirectoryName(target);
if (!string.IsNullOrEmpty(exportDirectory))
{
Directory.CreateDirectory(exportDirectory);
}
if (!ScriptRepoUpdater.Instance.ExportFileFromCenterRepo(normalized, target))
{
throw new FileNotFoundException($"仓库中不存在地图追踪文件: {normalized}");
}
return target;
}
// 兜底:镜像中不存在时按需写入
var content = ScriptRepoUpdater.Instance.ReadFileFromCenterRepo(normalized);
if (string.IsNullOrWhiteSpace(content))
@@ -503,6 +524,70 @@ public class GearTaskConverter
return target;
}
private void EnsurePathingRepoDirectoryExported(string repoRelativePath)
{
var normalized = repoRelativePath.Replace('\\', '/').Trim('/');
if (IsPathingRepoDirectoryExported(normalized))
{
return;
}
lock (_mirrorLock)
{
if (IsPathingRepoDirectoryExported(normalized))
{
return;
}
var targetDirectory = GetPathingMirrorPath(normalized);
if (string.Equals(normalized, "pathing", StringComparison.OrdinalIgnoreCase))
{
var mirrorRoot = GetPathingRepoMirrorRoot();
if (Directory.Exists(mirrorRoot))
{
Directory.Delete(mirrorRoot, true);
}
}
else if (Directory.Exists(targetDirectory))
{
Directory.Delete(targetDirectory, true);
}
Directory.CreateDirectory(targetDirectory);
ScriptRepoUpdater.Instance.ExportFilesFromCenterRepo(normalized, targetDirectory, ".json");
_exportedPathingRepoDirectories.Add(normalized);
}
}
private bool IsPathingRepoDirectoryExported(string repoRelativePath)
{
foreach (var exportedDirectory in _exportedPathingRepoDirectories)
{
if (string.Equals(exportedDirectory, repoRelativePath, StringComparison.OrdinalIgnoreCase)
|| repoRelativePath.StartsWith(exportedDirectory + "/", StringComparison.OrdinalIgnoreCase))
{
return true;
}
}
return false;
}
private static string GetPathingMirrorPath(string repoRelativePath)
{
var normalized = repoRelativePath.Replace('\\', '/').Trim('/');
var relativeUnderPathing = normalized.StartsWith("pathing/", StringComparison.OrdinalIgnoreCase)
? normalized["pathing/".Length..]
: normalized == "pathing"
? string.Empty
: normalized;
var mirrorRoot = GetPathingRepoMirrorRoot();
return string.IsNullOrEmpty(relativeUnderPathing)
? mirrorRoot
: Path.Combine(mirrorRoot, relativeUnderPathing.Replace('/', Path.DirectorySeparatorChar));
}
private void EnsurePathingRepoMirrorInitialized()
{
if (_pathingRepoMirrorInitialized)