From 57e8bc8bdf61d3fd7c0f99cad8d803c894eaf923 Mon Sep 17 00:00:00 2001 From: DismissedLight <1686188646@qq.com> Date: Mon, 4 Dec 2023 23:14:41 +0800 Subject: [PATCH] resource file sharding for client converting --- .../Core/ExceptionService/ThrowHelper.cs | 2 +- .../IO/Http/Sharding/HttpShardCopyWorker.cs | 168 ++++++++---------- .../Sharding/HttpShardCopyWorkerOptions.cs | 44 +++++ .../Game/Package/GamePackageService.cs | 51 +++--- .../Service/Game/Package/PackageConverter.cs | 73 +++----- .../ViewModel/Game/LaunchGameViewModel.cs | 6 +- 6 files changed, 175 insertions(+), 169 deletions(-) create mode 100644 src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorkerOptions.cs diff --git a/src/Snap.Hutao/Snap.Hutao/Core/ExceptionService/ThrowHelper.cs b/src/Snap.Hutao/Snap.Hutao/Core/ExceptionService/ThrowHelper.cs index c10f7373..4a1dfb0a 100644 --- a/src/Snap.Hutao/Snap.Hutao/Core/ExceptionService/ThrowHelper.cs +++ b/src/Snap.Hutao/Snap.Hutao/Core/ExceptionService/ThrowHelper.cs @@ -58,7 +58,7 @@ internal static class ThrowHelper [DoesNotReturn] [MethodImpl(MethodImplOptions.NoInlining)] - public static PackageConvertException PackageConvert(string message, Exception? inner) + public static PackageConvertException PackageConvert(string message, Exception? inner = default) { throw new PackageConvertException(message, inner); } diff --git a/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorker.cs b/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorker.cs index 91d8ac66..f8263402 100644 --- a/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorker.cs +++ b/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorker.cs @@ -3,7 +3,6 @@ using Microsoft.Win32.SafeHandles; using Snap.Hutao.Core.Diagnostics; -using Snap.Hutao.Web.Request.Builder; using System.IO; using System.Net.Http; @@ -15,46 +14,95 @@ internal sealed class HttpShardCopyWorker : IDisposable private readonly HttpClient httpClient; private readonly string sourceUrl; - private readonly Func statusFactory; + private readonly Func statusFactory; + private readonly long contentLength; private readonly int bufferSize; - private readonly SafeFileHandle destFileHandle; private readonly List shards; - private HttpShardCopyWorker(HttpClient httpClient, string sourceUrl, string destFilePath, long contentLength, Func statusFactory, int bufferSize) + private HttpShardCopyWorker(HttpShardCopyWorkerOptions options) { - this.httpClient = httpClient; - this.sourceUrl = sourceUrl; - this.statusFactory = statusFactory; - this.bufferSize = bufferSize; - - destFileHandle = File.OpenHandle( - destFilePath, - FileMode.Create, - FileAccess.Write, - FileShare.None, - FileOptions.RandomAccess | FileOptions.Asynchronous, - contentLength); - + httpClient = options.HttpClient; + sourceUrl = options.SourceUrl; + statusFactory = options.StatusFactory; + contentLength = options.ContentLength; + bufferSize = options.BufferSize; + destFileHandle = options.GetFileHandle(); shards = CalculateShards(contentLength); + + static List CalculateShards(long contentLength) + { + List shards = []; + long currentOffset = 0; + + while (currentOffset < contentLength) + { + long end = Math.Min(currentOffset + ShardSize, contentLength) - 1; + shards.Add(new Shard(currentOffset, end)); + currentOffset = end + 1; + } + + return shards; + } } - public static async ValueTask> CreateAsync(HttpClient httpClient, string sourceUrl, string destFilePath, Func statusFactory, int bufferSize = 81920) + public static async ValueTask> CreateAsync(HttpShardCopyWorkerOptions options) { - HttpResponseMessage response = await httpClient.HeadAsync(sourceUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - long contentLength = response.Content.Headers.ContentLength ?? 0; - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(contentLength); - - return new(httpClient, sourceUrl, destFilePath, contentLength, statusFactory, bufferSize); + await options.DetectContentLengthAsync().ConfigureAwait(false); + return new(options); } [SuppressMessage("", "SH003")] public Task CopyAsync(IProgress progress, CancellationToken token = default) { - ShardProgress shardProgress = new(progress, statusFactory); + ShardProgress shardProgress = new(progress, statusFactory, contentLength); return Parallel.ForEachAsync(shards, token, (shard, token) => CopyShardAsync(shard, shardProgress, token)); + + async ValueTask CopyShardAsync(Shard shard, IProgress progress, CancellationToken token) + { + ValueStopwatch stopwatch = ValueStopwatch.StartNew(); + HttpRequestMessage request = new(HttpMethod.Get, sourceUrl) + { + Headers = { Range = new(shard.StartOffset, shard.EndOffset), }, + }; + + using (request) + { + using (HttpResponseMessage response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token).ConfigureAwait(false)) + { + response.EnsureSuccessStatusCode(); + + Memory buffer = new byte[bufferSize]; + using (Stream stream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false)) + { + int totalBytesRead = 0; + int bytesReadAfterPreviousReport = 0; + do + { + int bytesRead = await stream.ReadAsync(buffer, token).ConfigureAwait(false); + if (bytesRead <= 0) + { + progress.Report(new(bytesReadAfterPreviousReport)); + bytesReadAfterPreviousReport = 0; + break; + } + + await RandomAccess.WriteAsync(destFileHandle, buffer[..bytesRead], shard.StartOffset + totalBytesRead, token).ConfigureAwait(false); + + totalBytesRead += bytesRead; + bytesReadAfterPreviousReport += bytesRead; + if (stopwatch.GetElapsedTime().TotalMilliseconds > 500) + { + progress.Report(new(bytesReadAfterPreviousReport)); + bytesReadAfterPreviousReport = 0; + stopwatch = ValueStopwatch.StartNew(); + } + } + while (true); + } + } + } + } } public void Dispose() @@ -62,66 +110,6 @@ internal sealed class HttpShardCopyWorker : IDisposable destFileHandle.Dispose(); } - private static List CalculateShards(long contentLength) - { - List shards = []; - long currentOffset = 0; - - while (currentOffset < contentLength) - { - long end = Math.Min(currentOffset + ShardSize, contentLength) - 1; - shards.Add(new Shard(currentOffset, end)); - currentOffset = end + 1; - } - - return shards; - } - - private async ValueTask CopyShardAsync(Shard shard, IProgress progress, CancellationToken token) - { - ValueStopwatch stopwatch = ValueStopwatch.StartNew(); - HttpRequestMessage request = new(HttpMethod.Get, sourceUrl) - { - Headers = - { - Range = new(shard.StartOffset, shard.EndOffset), - }, - }; - - using (request) - { - using (HttpResponseMessage response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token).ConfigureAwait(false)) - { - response.EnsureSuccessStatusCode(); - - Memory buffer = new byte[bufferSize]; - using (Stream stream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false)) - { - int totalBytesRead = 0; - do - { - int bytesRead = await stream.ReadAsync(buffer, token).ConfigureAwait(false); - if (bytesRead <= 0) - { - progress.Report(new(totalBytesRead)); - break; - } - - await RandomAccess.WriteAsync(destFileHandle, buffer[..bytesRead], shard.StartOffset + totalBytesRead, token).ConfigureAwait(false); - - totalBytesRead += bytesRead; - if (stopwatch.GetElapsedTime().TotalMilliseconds > 500) - { - progress.Report(new(totalBytesRead)); - stopwatch = ValueStopwatch.StartNew(); - } - } - while (true); - } - } - } - } - private sealed class Shard { public Shard(long startOffset, long endOffset) @@ -148,15 +136,17 @@ internal sealed class HttpShardCopyWorker : IDisposable private sealed class ShardProgress : IProgress { private readonly IProgress workerProgress; - private readonly Func statusFactory; + private readonly Func statusFactory; + private readonly long contentLength; private readonly object syncRoot = new(); private ValueStopwatch stopwatch = ValueStopwatch.StartNew(); private long totalBytesRead; - public ShardProgress(IProgress workerProgress, Func statusFactory) + public ShardProgress(IProgress workerProgress, Func statusFactory, long contentLength) { this.workerProgress = workerProgress; this.statusFactory = statusFactory; + this.contentLength = contentLength; } public void Report(ShardStatus value) @@ -168,7 +158,7 @@ internal sealed class HttpShardCopyWorker : IDisposable { if (stopwatch.GetElapsedTime().TotalMilliseconds > 500) { - workerProgress.Report(statusFactory(totalBytesRead)); + workerProgress.Report(statusFactory(totalBytesRead, contentLength)); stopwatch = ValueStopwatch.StartNew(); } } diff --git a/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorkerOptions.cs b/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorkerOptions.cs new file mode 100644 index 00000000..afa5ab83 --- /dev/null +++ b/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorkerOptions.cs @@ -0,0 +1,44 @@ +// Copyright (c) DGP Studio. All rights reserved. +// Licensed under the MIT license. + +using Microsoft.Win32.SafeHandles; +using System.IO; +using System.Net.Http; +using Snap.Hutao.Web.Request.Builder; + +namespace Snap.Hutao.Core.IO.Http.Sharding; + +internal sealed class HttpShardCopyWorkerOptions +{ + public HttpClient HttpClient { get; set; } = default!; + + public string SourceUrl { get; set; } = default!; + + public string DestinationFilePath { get; set; } = default!; + + public long ContentLength { get; private set; } + + public Func StatusFactory { get; set; } = default!; + + public int BufferSize { get; set; } = 80 * 1024; + + public SafeFileHandle GetFileHandle() + { + return File.OpenHandle(DestinationFilePath, FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.RandomAccess | FileOptions.Asynchronous, ContentLength); + } + + public async ValueTask DetectContentLengthAsync() + { + if (ContentLength > 0) + { + return; + } + + HttpResponseMessage response = await HttpClient.HeadAsync(SourceUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + long contentLength = response.Content.Headers.ContentLength ?? 0; + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(contentLength); + ContentLength = contentLength; + } +} \ No newline at end of file diff --git a/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/GamePackageService.cs b/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/GamePackageService.cs index 77b99c2b..7ab3e82c 100644 --- a/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/GamePackageService.cs +++ b/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/GamePackageService.cs @@ -31,37 +31,30 @@ internal sealed partial class GamePackageService : IGamePackageService .GetResourceAsync(launchScheme) .ConfigureAwait(false); - if (response.IsOk()) + if (!response.IsOk()) { - GameResource resource = response.Data; - - if (!launchScheme.ExecutableMatches(gameFileName)) - { - bool replaced = await packageConverter - .EnsureGameResourceAsync(launchScheme, resource, gameFolder, progress) - .ConfigureAwait(false); - - if (replaced) - { - // We need to change the gamePath if we switched. - string exeName = launchScheme.IsOversea ? GenshinImpactFileName : YuanShenFileName; - - await taskContext.SwitchToMainThreadAsync(); - appOptions.GamePath = Path.Combine(gameFolder, exeName); - } - else - { - // We can't start the game - // when we failed to convert game - return false; - } - } - - await packageConverter.EnsureDeprecatedFilesAndSdkAsync(resource, gameFolder).ConfigureAwait(false); - - return true; + return false; } - return false; + GameResource resource = response.Data; + + if (!launchScheme.ExecutableMatches(gameFileName)) + { + // We can't start the game + // when we failed to convert game + if (!await packageConverter.EnsureGameResourceAsync(launchScheme, resource, gameFolder, progress).ConfigureAwait(false)) + { + return false; + } + + // We need to change the gamePath if we switched. + string exeName = launchScheme.IsOversea ? GenshinImpactFileName : YuanShenFileName; + + await taskContext.SwitchToMainThreadAsync(); + appOptions.GamePath = Path.Combine(gameFolder, exeName); + } + + await packageConverter.EnsureDeprecatedFilesAndSdkAsync(resource, gameFolder).ConfigureAwait(false); + return true; } } \ No newline at end of file diff --git a/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/PackageConverter.cs b/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/PackageConverter.cs index 65098687..0c20d0bf 100644 --- a/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/PackageConverter.cs +++ b/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/PackageConverter.cs @@ -6,6 +6,7 @@ using Snap.Hutao.Core.DependencyInjection.Annotation.HttpClient; using Snap.Hutao.Core.ExceptionService; using Snap.Hutao.Core.IO; using Snap.Hutao.Core.IO.Hashing; +using Snap.Hutao.Core.IO.Http.Sharding; using Snap.Hutao.Service.Game.Scheme; using Snap.Hutao.Web.Hoyolab.SdkStatic.Hk4e.Launcher; using System.Globalization; @@ -31,15 +32,6 @@ internal sealed partial class PackageConverter private readonly HttpClient httpClient; private readonly ILogger logger; - /// - /// 异步检查替换游戏资源 - /// 调用前需要确认本地文件与服务器上的不同 - /// - /// 目标启动方案 - /// 游戏资源 - /// 游戏目录 - /// 进度 - /// 替换结果与资源 public async ValueTask EnsureGameResourceAsync(LaunchScheme targetScheme, GameResource gameResource, string gameFolder, IProgress progress) { // 以 国服 => 国际 为例 @@ -83,13 +75,6 @@ internal sealed partial class PackageConverter return await ReplaceGameResourceAsync(diffOperations, context, progress).ConfigureAwait(false); } - /// - /// 检查过时文件与Sdk - /// 只在国服环境有效 - /// - /// 游戏资源 - /// 游戏文件夹 - /// 任务 public async ValueTask EnsureDeprecatedFilesAndSdkAsync(GameResource resource, string gameFolder) { string sdkDllBackup = Path.Combine(gameFolder, YuanShenData, "Plugins\\PCGameSDK.dll.backup"); @@ -182,12 +167,11 @@ internal sealed partial class PackageConverter Dictionary results = []; using (StreamReader reader = new(stream)) { - Regex dataFolderRegex = DataFolderRegex(); while (await reader.ReadLineAsync().ConfigureAwait(false) is { Length: > 0 } row) { VersionItem? item = JsonSerializer.Deserialize(row, options); ArgumentNullException.ThrowIfNull(item); - item.RelativePath = dataFolderRegex.Replace(item.RelativePath, "{0}"); + item.RelativePath = DataFolderRegex().Replace(item.RelativePath, "{0}"); results.Add(item.RelativePath, item); } } @@ -244,8 +228,7 @@ internal sealed partial class PackageConverter { if (info.Remote.FileSize == new FileInfo(cacheFile).Length) { - string cacheMd5 = await MD5.HashFileAsync(cacheFile).ConfigureAwait(false); - if (info.Remote.Md5.Equals(cacheMd5, StringComparison.OrdinalIgnoreCase)) + if (info.Remote.Md5.Equals(await MD5.HashFileAsync(cacheFile).ConfigureAwait(false), StringComparison.OrdinalIgnoreCase)) { return; } @@ -259,35 +242,33 @@ internal sealed partial class PackageConverter string? directory = Path.GetDirectoryName(cacheFile); ArgumentException.ThrowIfNullOrEmpty(directory); Directory.CreateDirectory(directory); - using (FileStream fileStream = File.Create(cacheFile)) + + string remoteUrl = context.GetScatteredFilesUrl(remoteName); + HttpShardCopyWorkerOptions options = new() { - string remoteUrl = context.GetScatteredFilesUrl(remoteName); - using (HttpResponseMessage response = await httpClient.GetAsync(remoteUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false)) + HttpClient = httpClient, + SourceUrl = remoteUrl, + DestinationFilePath = cacheFile, + StatusFactory = (bytesRead, totalBytes) => new(remoteName, bytesRead, totalBytes), + }; + + using (HttpShardCopyWorker worker = await HttpShardCopyWorker.CreateAsync(options).ConfigureAwait(false)) + { + try { - // This stream's length is incorrect, - // so we use length in the header - long totalBytes = response.Content.Headers.ContentLength ?? 0; - using (Stream webStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) - { - try - { - StreamCopyWorker streamCopyWorker = new(webStream, fileStream, bytesRead => new(remoteName, bytesRead, totalBytes)); - await streamCopyWorker.CopyAsync(progress).ConfigureAwait(false); - fileStream.Position = 0; - string cacheMd5 = await MD5.HashAsync(fileStream).ConfigureAwait(false); - if (string.Equals(info.Remote.Md5, cacheMd5, StringComparison.OrdinalIgnoreCase)) - { - return; - } - } - catch (Exception ex) - { - // System.IO.IOException: The response ended prematurely. - // System.IO.IOException: Received an unexpected EOF or 0 bytes from the transport stream. - ThrowHelper.PackageConvert(SH.FormatServiceGamePackageRequestScatteredFileFailed(remoteName), ex); - } - } + await worker.CopyAsync(progress).ConfigureAwait(false); } + catch (Exception ex) + { + // System.IO.IOException: The response ended prematurely. + // System.IO.IOException: Received an unexpected EOF or 0 bytes from the transport stream. + ThrowHelper.PackageConvert(SH.FormatServiceGamePackageRequestScatteredFileFailed(remoteName), ex); + } + } + + if (!string.Equals(info.Remote.Md5, await MD5.HashFileAsync(cacheFile).ConfigureAwait(false), StringComparison.OrdinalIgnoreCase)) + { + ThrowHelper.PackageConvert(SH.FormatServiceGamePackageRequestScatteredFileFailed(remoteName)); } } diff --git a/src/Snap.Hutao/Snap.Hutao/ViewModel/Game/LaunchGameViewModel.cs b/src/Snap.Hutao/Snap.Hutao/ViewModel/Game/LaunchGameViewModel.cs index c80f600a..a33cf0f8 100644 --- a/src/Snap.Hutao/Snap.Hutao/ViewModel/Game/LaunchGameViewModel.cs +++ b/src/Snap.Hutao/Snap.Hutao/ViewModel/Game/LaunchGameViewModel.cs @@ -123,9 +123,7 @@ internal sealed partial class LaunchGameViewModel : Abstraction.ViewModel { try { - SelectedScheme = KnownSchemes - .Where(scheme => scheme.IsOversea == options.IsOversea) - .Single(scheme => scheme.Equals(options)); + SelectedScheme = KnownSchemes.Single(scheme => scheme.Equals(options)); } catch (InvalidOperationException) { @@ -201,7 +199,6 @@ internal sealed partial class LaunchGameViewModel : Abstraction.ViewModel try { - // Always ensure game resources gameService.SetChannelOptions(SelectedScheme); LaunchGamePackageConvertDialog dialog = await contentDialogFactory.CreateInstanceAsync().ConfigureAwait(false); @@ -209,6 +206,7 @@ internal sealed partial class LaunchGameViewModel : Abstraction.ViewModel using (await dialog.BlockAsync(taskContext).ConfigureAwait(false)) { + // Always ensure game resources if (!await gameService.EnsureGameResourceAsync(SelectedScheme, convertProgress).ConfigureAwait(false)) { infoBarService.Warning(SH.ViewModelLaunchGameEnsureGameResourceFail);