resource file sharding for client converting

2025-11-19 21:02:53 +08:00 · 2023-12-04 23:14:41 +08:00
parent b6ad96c0cb
commit 57e8bc8bdf
6 changed files with 175 additions and 169 deletions
--- a/src/Snap.Hutao/Snap.Hutao/Core/ExceptionService/ThrowHelper.cs
+++ b/src/Snap.Hutao/Snap.Hutao/Core/ExceptionService/ThrowHelper.cs
@@ -58,7 +58,7 @@ internal static class ThrowHelper

    [DoesNotReturn]
    [MethodImpl(MethodImplOptions.NoInlining)]
-    public static PackageConvertException PackageConvert(string message, Exception? inner)
+    public static PackageConvertException PackageConvert(string message, Exception? inner = default)
    {
        throw new PackageConvertException(message, inner);
    }
--- a/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorker.cs
+++ b/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorker.cs
@@ -3,7 +3,6 @@

 using Microsoft.Win32.SafeHandles;
 using Snap.Hutao.Core.Diagnostics;
-using Snap.Hutao.Web.Request.Builder;
 using System.IO;
 using System.Net.Http;

@@ -15,46 +14,95 @@ internal sealed class HttpShardCopyWorker<TStatus> : IDisposable

    private readonly HttpClient httpClient;
    private readonly string sourceUrl;
-    private readonly Func<long, TStatus> statusFactory;
+    private readonly Func<long, long, TStatus> statusFactory;
+    private readonly long contentLength;
    private readonly int bufferSize;
-
    private readonly SafeFileHandle destFileHandle;
    private readonly List<Shard> shards;

-    private HttpShardCopyWorker(HttpClient httpClient, string sourceUrl, string destFilePath, long contentLength, Func<long, TStatus> statusFactory, int bufferSize)
+    private HttpShardCopyWorker(HttpShardCopyWorkerOptions<TStatus> options)
    {
-        this.httpClient = httpClient;
-        this.sourceUrl = sourceUrl;
-        this.statusFactory = statusFactory;
-        this.bufferSize = bufferSize;
-
-        destFileHandle = File.OpenHandle(
-            destFilePath,
-            FileMode.Create,
-            FileAccess.Write,
-            FileShare.None,
-            FileOptions.RandomAccess | FileOptions.Asynchronous,
-            contentLength);
-
+        httpClient = options.HttpClient;
+        sourceUrl = options.SourceUrl;
+        statusFactory = options.StatusFactory;
+        contentLength = options.ContentLength;
+        bufferSize = options.BufferSize;
+        destFileHandle = options.GetFileHandle();
        shards = CalculateShards(contentLength);
+
+        static List<Shard> CalculateShards(long contentLength)
+        {
+            List<Shard> shards = [];
+            long currentOffset = 0;
+
+            while (currentOffset < contentLength)
+            {
+                long end = Math.Min(currentOffset + ShardSize, contentLength) - 1;
+                shards.Add(new Shard(currentOffset, end));
+                currentOffset = end + 1;
+            }
+
+            return shards;
+        }
    }

-    public static async ValueTask<HttpShardCopyWorker<TStatus>> CreateAsync(HttpClient httpClient, string sourceUrl, string destFilePath, Func<long, TStatus> statusFactory, int bufferSize = 81920)
+    public static async ValueTask<HttpShardCopyWorker<TStatus>> CreateAsync(HttpShardCopyWorkerOptions<TStatus> options)
    {
-        HttpResponseMessage response = await httpClient.HeadAsync(sourceUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
-        response.EnsureSuccessStatusCode();
-
-        long contentLength = response.Content.Headers.ContentLength ?? 0;
-        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(contentLength);
-
-        return new(httpClient, sourceUrl, destFilePath, contentLength, statusFactory, bufferSize);
+        await options.DetectContentLengthAsync().ConfigureAwait(false);
+        return new(options);
    }

    [SuppressMessage("", "SH003")]
    public Task CopyAsync(IProgress<TStatus> progress, CancellationToken token = default)
    {
-        ShardProgress shardProgress = new(progress, statusFactory);
+        ShardProgress shardProgress = new(progress, statusFactory, contentLength);
        return Parallel.ForEachAsync(shards, token, (shard, token) => CopyShardAsync(shard, shardProgress, token));
+
+        async ValueTask CopyShardAsync(Shard shard, IProgress<ShardStatus> progress, CancellationToken token)
+        {
+            ValueStopwatch stopwatch = ValueStopwatch.StartNew();
+            HttpRequestMessage request = new(HttpMethod.Get, sourceUrl)
+            {
+                Headers = { Range = new(shard.StartOffset, shard.EndOffset), },
+            };
+
+            using (request)
+            {
+                using (HttpResponseMessage response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token).ConfigureAwait(false))
+                {
+                    response.EnsureSuccessStatusCode();
+
+                    Memory<byte> buffer = new byte[bufferSize];
+                    using (Stream stream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false))
+                    {
+                        int totalBytesRead = 0;
+                        int bytesReadAfterPreviousReport = 0;
+                        do
+                        {
+                            int bytesRead = await stream.ReadAsync(buffer, token).ConfigureAwait(false);
+                            if (bytesRead <= 0)
+                            {
+                                progress.Report(new(bytesReadAfterPreviousReport));
+                                bytesReadAfterPreviousReport = 0;
+                                break;
+                            }
+
+                            await RandomAccess.WriteAsync(destFileHandle, buffer[..bytesRead], shard.StartOffset + totalBytesRead, token).ConfigureAwait(false);
+
+                            totalBytesRead += bytesRead;
+                            bytesReadAfterPreviousReport += bytesRead;
+                            if (stopwatch.GetElapsedTime().TotalMilliseconds > 500)
+                            {
+                                progress.Report(new(bytesReadAfterPreviousReport));
+                                bytesReadAfterPreviousReport = 0;
+                                stopwatch = ValueStopwatch.StartNew();
+                            }
+                        }
+                        while (true);
+                    }
+                }
+            }
+        }
    }

    public void Dispose()
@@ -62,66 +110,6 @@ internal sealed class HttpShardCopyWorker<TStatus> : IDisposable
        destFileHandle.Dispose();
    }

-    private static List<Shard> CalculateShards(long contentLength)
-    {
-        List<Shard> shards = [];
-        long currentOffset = 0;
-
-        while (currentOffset < contentLength)
-        {
-            long end = Math.Min(currentOffset + ShardSize, contentLength) - 1;
-            shards.Add(new Shard(currentOffset, end));
-            currentOffset = end + 1;
-        }
-
-        return shards;
-    }
-
-    private async ValueTask CopyShardAsync(Shard shard, IProgress<ShardStatus> progress, CancellationToken token)
-    {
-        ValueStopwatch stopwatch = ValueStopwatch.StartNew();
-        HttpRequestMessage request = new(HttpMethod.Get, sourceUrl)
-        {
-            Headers =
-            {
-                Range = new(shard.StartOffset, shard.EndOffset),
-            },
-        };
-
-        using (request)
-        {
-            using (HttpResponseMessage response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token).ConfigureAwait(false))
-            {
-                response.EnsureSuccessStatusCode();
-
-                Memory<byte> buffer = new byte[bufferSize];
-                using (Stream stream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false))
-                {
-                    int totalBytesRead = 0;
-                    do
-                    {
-                        int bytesRead = await stream.ReadAsync(buffer, token).ConfigureAwait(false);
-                        if (bytesRead <= 0)
-                        {
-                            progress.Report(new(totalBytesRead));
-                            break;
-                        }
-
-                        await RandomAccess.WriteAsync(destFileHandle, buffer[..bytesRead], shard.StartOffset + totalBytesRead, token).ConfigureAwait(false);
-
-                        totalBytesRead += bytesRead;
-                        if (stopwatch.GetElapsedTime().TotalMilliseconds > 500)
-                        {
-                            progress.Report(new(totalBytesRead));
-                            stopwatch = ValueStopwatch.StartNew();
-                        }
-                    }
-                    while (true);
-                }
-            }
-        }
-    }
-
    private sealed class Shard
    {
        public Shard(long startOffset, long endOffset)
@@ -148,15 +136,17 @@ internal sealed class HttpShardCopyWorker<TStatus> : IDisposable
    private sealed class ShardProgress : IProgress<ShardStatus>
    {
        private readonly IProgress<TStatus> workerProgress;
-        private readonly Func<long, TStatus> statusFactory;
+        private readonly Func<long, long, TStatus> statusFactory;
+        private readonly long contentLength;
        private readonly object syncRoot = new();
        private ValueStopwatch stopwatch = ValueStopwatch.StartNew();
        private long totalBytesRead;

-        public ShardProgress(IProgress<TStatus> workerProgress, Func<long, TStatus> statusFactory)
+        public ShardProgress(IProgress<TStatus> workerProgress, Func<long, long, TStatus> statusFactory, long contentLength)
        {
            this.workerProgress = workerProgress;
            this.statusFactory = statusFactory;
+            this.contentLength = contentLength;
        }

        public void Report(ShardStatus value)
@@ -168,7 +158,7 @@ internal sealed class HttpShardCopyWorker<TStatus> : IDisposable
                {
                    if (stopwatch.GetElapsedTime().TotalMilliseconds > 500)
                    {
-                        workerProgress.Report(statusFactory(totalBytesRead));
+                        workerProgress.Report(statusFactory(totalBytesRead, contentLength));
                        stopwatch = ValueStopwatch.StartNew();
                    }
                }
--- a/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorkerOptions.cs
+++ b/src/Snap.Hutao/Snap.Hutao/Core/IO/Http/Sharding/HttpShardCopyWorkerOptions.cs
@@ -0,0 +1,44 @@
+// Copyright (c) DGP Studio. All rights reserved.
+// Licensed under the MIT license.
+
+using Microsoft.Win32.SafeHandles;
+using System.IO;
+using System.Net.Http;
+using Snap.Hutao.Web.Request.Builder;
+
+namespace Snap.Hutao.Core.IO.Http.Sharding;
+
+internal sealed class HttpShardCopyWorkerOptions<TStatus>
+{
+    public HttpClient HttpClient { get; set; } = default!;
+
+    public string SourceUrl { get; set; } = default!;
+
+    public string DestinationFilePath { get; set; } = default!;
+
+    public long ContentLength { get; private set; }
+
+    public Func<long, long, TStatus> StatusFactory { get; set; } = default!;
+
+    public int BufferSize { get; set; } = 80 * 1024;
+
+    public SafeFileHandle GetFileHandle()
+    {
+        return File.OpenHandle(DestinationFilePath, FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.RandomAccess | FileOptions.Asynchronous, ContentLength);
+    }
+
+    public async ValueTask DetectContentLengthAsync()
+    {
+        if (ContentLength > 0)
+        {
+            return;
+        }
+
+        HttpResponseMessage response = await HttpClient.HeadAsync(SourceUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
+        response.EnsureSuccessStatusCode();
+
+        long contentLength = response.Content.Headers.ContentLength ?? 0;
+        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(contentLength);
+        ContentLength = contentLength;
+    }
+}
--- a/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/GamePackageService.cs
+++ b/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/GamePackageService.cs
@@ -31,37 +31,30 @@ internal sealed partial class GamePackageService : IGamePackageService
            .GetResourceAsync(launchScheme)
            .ConfigureAwait(false);

-        if (response.IsOk())
+        if (!response.IsOk())
        {
-            GameResource resource = response.Data;
-
-            if (!launchScheme.ExecutableMatches(gameFileName))
-            {
-                bool replaced = await packageConverter
-                    .EnsureGameResourceAsync(launchScheme, resource, gameFolder, progress)
-                    .ConfigureAwait(false);
-
-                if (replaced)
-                {
-                    // We need to change the gamePath if we switched.
-                    string exeName = launchScheme.IsOversea ? GenshinImpactFileName : YuanShenFileName;
-
-                    await taskContext.SwitchToMainThreadAsync();
-                    appOptions.GamePath = Path.Combine(gameFolder, exeName);
-                }
-                else
-                {
-                    // We can't start the game
-                    // when we failed to convert game
-                    return false;
-                }
-            }
-
-            await packageConverter.EnsureDeprecatedFilesAndSdkAsync(resource, gameFolder).ConfigureAwait(false);
-
-            return true;
+            return false;
        }

-        return false;
+        GameResource resource = response.Data;
+
+        if (!launchScheme.ExecutableMatches(gameFileName))
+        {
+            // We can't start the game
+            // when we failed to convert game
+            if (!await packageConverter.EnsureGameResourceAsync(launchScheme, resource, gameFolder, progress).ConfigureAwait(false))
+            {
+                return false;
+            }
+
+            // We need to change the gamePath if we switched.
+            string exeName = launchScheme.IsOversea ? GenshinImpactFileName : YuanShenFileName;
+
+            await taskContext.SwitchToMainThreadAsync();
+            appOptions.GamePath = Path.Combine(gameFolder, exeName);
+        }
+
+        await packageConverter.EnsureDeprecatedFilesAndSdkAsync(resource, gameFolder).ConfigureAwait(false);
+        return true;
    }
 }
--- a/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/PackageConverter.cs
+++ b/src/Snap.Hutao/Snap.Hutao/Service/Game/Package/PackageConverter.cs
@@ -6,6 +6,7 @@ using Snap.Hutao.Core.DependencyInjection.Annotation.HttpClient;
 using Snap.Hutao.Core.ExceptionService;
 using Snap.Hutao.Core.IO;
 using Snap.Hutao.Core.IO.Hashing;
+using Snap.Hutao.Core.IO.Http.Sharding;
 using Snap.Hutao.Service.Game.Scheme;
 using Snap.Hutao.Web.Hoyolab.SdkStatic.Hk4e.Launcher;
 using System.Globalization;
@@ -31,15 +32,6 @@ internal sealed partial class PackageConverter
    private readonly HttpClient httpClient;
    private readonly ILogger<PackageConverter> logger;

-    /// <summary>
-    /// 异步检查替换游戏资源
-    /// 调用前需要确认本地文件与服务器上的不同
-    /// </summary>
-    /// <param name="targetScheme">目标启动方案</param>
-    /// <param name="gameResource">游戏资源</param>
-    /// <param name="gameFolder">游戏目录</param>
-    /// <param name="progress">进度</param>
-    /// <returns>替换结果与资源</returns>
    public async ValueTask<bool> EnsureGameResourceAsync(LaunchScheme targetScheme, GameResource gameResource, string gameFolder, IProgress<PackageReplaceStatus> progress)
    {
        // 以 国服 => 国际 为例
@@ -83,13 +75,6 @@ internal sealed partial class PackageConverter
        return await ReplaceGameResourceAsync(diffOperations, context, progress).ConfigureAwait(false);
    }

-    /// <summary>
-    /// 检查过时文件与Sdk
-    /// 只在国服环境有效
-    /// </summary>
-    /// <param name="resource">游戏资源</param>
-    /// <param name="gameFolder">游戏文件夹</param>
-    /// <returns>任务</returns>
    public async ValueTask EnsureDeprecatedFilesAndSdkAsync(GameResource resource, string gameFolder)
    {
        string sdkDllBackup = Path.Combine(gameFolder, YuanShenData, "Plugins\\PCGameSDK.dll.backup");
@@ -182,12 +167,11 @@ internal sealed partial class PackageConverter
        Dictionary<string, VersionItem> results = [];
        using (StreamReader reader = new(stream))
        {
-            Regex dataFolderRegex = DataFolderRegex();
            while (await reader.ReadLineAsync().ConfigureAwait(false) is { Length: > 0 } row)
            {
                VersionItem? item = JsonSerializer.Deserialize<VersionItem>(row, options);
                ArgumentNullException.ThrowIfNull(item);
-                item.RelativePath = dataFolderRegex.Replace(item.RelativePath, "{0}");
+                item.RelativePath = DataFolderRegex().Replace(item.RelativePath, "{0}");
                results.Add(item.RelativePath, item);
            }
        }
@@ -244,8 +228,7 @@ internal sealed partial class PackageConverter
        {
            if (info.Remote.FileSize == new FileInfo(cacheFile).Length)
            {
-                string cacheMd5 = await MD5.HashFileAsync(cacheFile).ConfigureAwait(false);
-                if (info.Remote.Md5.Equals(cacheMd5, StringComparison.OrdinalIgnoreCase))
+                if (info.Remote.Md5.Equals(await MD5.HashFileAsync(cacheFile).ConfigureAwait(false), StringComparison.OrdinalIgnoreCase))
                {
                    return;
                }
@@ -259,35 +242,33 @@ internal sealed partial class PackageConverter
        string? directory = Path.GetDirectoryName(cacheFile);
        ArgumentException.ThrowIfNullOrEmpty(directory);
        Directory.CreateDirectory(directory);
-        using (FileStream fileStream = File.Create(cacheFile))
+
+        string remoteUrl = context.GetScatteredFilesUrl(remoteName);
+        HttpShardCopyWorkerOptions<PackageReplaceStatus> options = new()
        {
-            string remoteUrl = context.GetScatteredFilesUrl(remoteName);
-            using (HttpResponseMessage response = await httpClient.GetAsync(remoteUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false))
+            HttpClient = httpClient,
+            SourceUrl = remoteUrl,
+            DestinationFilePath = cacheFile,
+            StatusFactory = (bytesRead, totalBytes) => new(remoteName, bytesRead, totalBytes),
+        };
+
+        using (HttpShardCopyWorker<PackageReplaceStatus> worker = await HttpShardCopyWorker<PackageReplaceStatus>.CreateAsync(options).ConfigureAwait(false))
+        {
+            try
            {
-                // This stream's length is incorrect,
-                // so we use length in the header
-                long totalBytes = response.Content.Headers.ContentLength ?? 0;
-                using (Stream webStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
-                {
-                    try
-                    {
-                        StreamCopyWorker<PackageReplaceStatus> streamCopyWorker = new(webStream, fileStream, bytesRead => new(remoteName, bytesRead, totalBytes));
-                        await streamCopyWorker.CopyAsync(progress).ConfigureAwait(false);
-                        fileStream.Position = 0;
-                        string cacheMd5 = await MD5.HashAsync(fileStream).ConfigureAwait(false);
-                        if (string.Equals(info.Remote.Md5, cacheMd5, StringComparison.OrdinalIgnoreCase))
-                        {
-                            return;
-                        }
-                    }
-                    catch (Exception ex)
-                    {
-                        // System.IO.IOException: The response ended prematurely.
-                        // System.IO.IOException: Received an unexpected EOF or 0 bytes from the transport stream.
-                        ThrowHelper.PackageConvert(SH.FormatServiceGamePackageRequestScatteredFileFailed(remoteName), ex);
-                    }
-                }
+                await worker.CopyAsync(progress).ConfigureAwait(false);
            }
+            catch (Exception ex)
+            {
+                // System.IO.IOException: The response ended prematurely.
+                // System.IO.IOException: Received an unexpected EOF or 0 bytes from the transport stream.
+                ThrowHelper.PackageConvert(SH.FormatServiceGamePackageRequestScatteredFileFailed(remoteName), ex);
+            }
+        }
+
+        if (!string.Equals(info.Remote.Md5, await MD5.HashFileAsync(cacheFile).ConfigureAwait(false), StringComparison.OrdinalIgnoreCase))
+        {
+            ThrowHelper.PackageConvert(SH.FormatServiceGamePackageRequestScatteredFileFailed(remoteName));
        }
    }

--- a/src/Snap.Hutao/Snap.Hutao/ViewModel/Game/LaunchGameViewModel.cs
+++ b/src/Snap.Hutao/Snap.Hutao/ViewModel/Game/LaunchGameViewModel.cs
@@ -123,9 +123,7 @@ internal sealed partial class LaunchGameViewModel : Abstraction.ViewModel
                    {
                        try
                        {
-                            SelectedScheme = KnownSchemes
-                                .Where(scheme => scheme.IsOversea == options.IsOversea)
-                                .Single(scheme => scheme.Equals(options));
+                            SelectedScheme = KnownSchemes.Single(scheme => scheme.Equals(options));
                        }
                        catch (InvalidOperationException)
                        {
@@ -201,7 +199,6 @@ internal sealed partial class LaunchGameViewModel : Abstraction.ViewModel

        try
        {
-            // Always ensure game resources
            gameService.SetChannelOptions(SelectedScheme);

            LaunchGamePackageConvertDialog dialog = await contentDialogFactory.CreateInstanceAsync<LaunchGamePackageConvertDialog>().ConfigureAwait(false);
@@ -209,6 +206,7 @@ internal sealed partial class LaunchGameViewModel : Abstraction.ViewModel

            using (await dialog.BlockAsync(taskContext).ConfigureAwait(false))
            {
+                // Always ensure game resources
                if (!await gameService.EnsureGameResourceAsync(SelectedScheme, convertProgress).ConfigureAwait(false))
                {
                    infoBarService.Warning(SH.ViewModelLaunchGameEnsureGameResourceFail);