From a718ba16e2fde5d5da2bc3a00f8cac64a4333b80 Mon Sep 17 00:00:00 2001 From: DismissedLight <1686188646@qq.com> Date: Sun, 20 Nov 2022 14:54:04 +0800 Subject: [PATCH] introduce cache download retry and host replace --- .../Snap.Hutao/Core/Caching/CacheBase.cs | 246 ------------------ .../Snap.Hutao/Core/Caching/ImageCache.cs | 239 ++++++++++++++++- 2 files changed, 235 insertions(+), 250 deletions(-) delete mode 100644 src/Snap.Hutao/Snap.Hutao/Core/Caching/CacheBase.cs diff --git a/src/Snap.Hutao/Snap.Hutao/Core/Caching/CacheBase.cs b/src/Snap.Hutao/Snap.Hutao/Core/Caching/CacheBase.cs deleted file mode 100644 index e0643b4a..00000000 --- a/src/Snap.Hutao/Snap.Hutao/Core/Caching/CacheBase.cs +++ /dev/null @@ -1,246 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Snap.Hutao.Core.Logging; -using System.IO; -using System.Net.Http; -using System.Security.Cryptography; -using System.Text; -using Windows.Storage; -using Windows.Storage.FileProperties; - -namespace Snap.Hutao.Core.Caching; - -/// -/// Provides methods and tools to cache files in a folder -/// 经过简化 -/// -/// Generic type as supplied by consumer of the class -[SuppressMessage("", "CA1001")] -public abstract class CacheBase - where T : class -{ - private readonly SemaphoreSlim cacheFolderSemaphore = new(1); - private readonly ILogger logger; - - // violate di rule - private readonly HttpClient httpClient; - - private StorageFolder? baseFolder; - private string? cacheFolderName; - private StorageFolder? cacheFolder; - - /// - /// Initializes a new instance of the class. - /// - /// 日志器 - /// http客户端 - protected CacheBase(ILogger logger, HttpClient httpClient) - { - this.logger = logger; - this.httpClient = httpClient; - - CacheDuration = TimeSpan.FromDays(30); - RetryCount = 3; - } - - /// - /// Gets or sets the life duration of every cache entry. - /// - public TimeSpan CacheDuration { get; } - - /// - /// Gets or sets the number of retries trying to ensure the file is cached. - /// - public uint RetryCount { get; } - - /// - /// Clears all files in the cache - /// - /// awaitable task - public async Task ClearAsync() - { - StorageFolder folder = await GetCacheFolderAsync().ConfigureAwait(false); - IReadOnlyList files = await folder.GetFilesAsync().AsTask().ConfigureAwait(false); - - await RemoveAsync(files).ConfigureAwait(false); - } - - /// - /// Removes cached files that have expired - /// - /// Optional timespan to compute whether file has expired or not. If no value is supplied, is used. - /// awaitable task - public async Task RemoveExpiredAsync(TimeSpan? duration = null) - { - TimeSpan expiryDuration = duration ?? CacheDuration; - - StorageFolder folder = await GetCacheFolderAsync().ConfigureAwait(false); - IReadOnlyList files = await folder.GetFilesAsync().AsTask().ConfigureAwait(false); - - List filesToDelete = new(); - - foreach (StorageFile file in files) - { - if (file == null) - { - continue; - } - - if (await IsFileOutOfDateAsync(file, expiryDuration, false).ConfigureAwait(false)) - { - filesToDelete.Add(file); - } - } - - await RemoveAsync(filesToDelete).ConfigureAwait(false); - } - - /// - /// Removed items based on uri list passed - /// - /// Enumerable uri list - /// awaitable Task - public async Task RemoveAsync(IEnumerable uriForCachedItems) - { - if (uriForCachedItems == null || !uriForCachedItems.Any()) - { - return; - } - - StorageFolder folder = await GetCacheFolderAsync().ConfigureAwait(false); - IReadOnlyList files = await folder.GetFilesAsync().AsTask().ConfigureAwait(false); - - List filesToDelete = new(); - - Dictionary cachedFiles = files.ToDictionary(file => file.Name); - - foreach (Uri uri in uriForCachedItems) - { - string fileName = GetCacheFileName(uri); - if (cachedFiles.TryGetValue(fileName, out StorageFile? file)) - { - filesToDelete.Add(file); - } - } - - await RemoveAsync(filesToDelete).ConfigureAwait(false); - } - - /// - /// Gets the StorageFile containing cached item for given Uri - /// - /// Uri of the item. - /// a StorageFile - public async Task GetFileFromCacheAsync(Uri uri) - { - StorageFolder folder = await GetCacheFolderAsync().ConfigureAwait(false); - - string fileName = GetCacheFileName(uri); - - IStorageItem? item = await folder.TryGetItemAsync(fileName).AsTask().ConfigureAwait(false); - - if (item == null || (await item.GetBasicPropertiesAsync()).Size == 0) - { - StorageFile baseFile = await folder.CreateFileAsync(fileName, CreationCollisionOption.ReplaceExisting).AsTask().ConfigureAwait(false); - await DownloadFileAsync(uri, baseFile).ConfigureAwait(false); - item = await folder.TryGetItemAsync(fileName).AsTask().ConfigureAwait(false); - } - - return Must.NotNull((item as StorageFile)!); - } - - /// - /// Override-able method that checks whether file is valid or not. - /// - /// storage file - /// cache duration - /// option to mark uninitialized file as expired - /// bool indicate whether file has expired or not - protected virtual async Task IsFileOutOfDateAsync(StorageFile file, TimeSpan duration, bool treatNullFileAsOutOfDate = true) - { - if (file == null) - { - return treatNullFileAsOutOfDate; - } - - BasicProperties? properties = await file.GetBasicPropertiesAsync().AsTask().ConfigureAwait(false); - - return properties.Size == 0 || DateTime.Now.Subtract(properties.DateModified.DateTime) > duration; - } - - private static string GetCacheFileName(Uri uri) - { - string url = uri.ToString(); - byte[] chars = Encoding.UTF8.GetBytes(url); - byte[] hash = SHA1.HashData(chars); - return System.Convert.ToHexString(hash); - } - - private async Task DownloadFileAsync(Uri uri, StorageFile baseFile) - { - logger.LogInformation(EventIds.FileCaching, "Begin downloading for {uri}", uri); - - using (Stream httpStream = await httpClient.GetStreamAsync(uri).ConfigureAwait(false)) - { - using (FileStream fileStream = File.Create(baseFile.Path)) - { - await httpStream.CopyToAsync(fileStream).ConfigureAwait(false); - } - } - } - - /// - /// Initializes with default values if user has not initialized explicitly - /// - /// awaitable task - private async Task InitializeInternalAsync() - { - if (cacheFolder != null) - { - return; - } - - using (await cacheFolderSemaphore.EnterAsync().ConfigureAwait(false)) - { - baseFolder ??= ApplicationData.Current.TemporaryFolder; - - if (string.IsNullOrWhiteSpace(cacheFolderName)) - { - cacheFolderName = GetType().Name; - } - - cacheFolder = await baseFolder - .CreateFolderAsync(cacheFolderName, CreationCollisionOption.OpenIfExists) - .AsTask() - .ConfigureAwait(false); - } - } - - private async Task GetCacheFolderAsync() - { - if (cacheFolder == null) - { - await InitializeInternalAsync().ConfigureAwait(false); - } - - return Must.NotNull(cacheFolder!); - } - - private async Task RemoveAsync(IEnumerable files) - { - foreach (StorageFile file in files) - { - try - { - logger.LogInformation(EventIds.CacheRemoveFile, "Removing file {file}", file.Path); - await file.DeleteAsync().AsTask().ConfigureAwait(false); - } - catch - { - logger.LogError(EventIds.CacheException, "Failed to delete file: {file}", file.Path); - } - } - } -} \ No newline at end of file diff --git a/src/Snap.Hutao/Snap.Hutao/Core/Caching/ImageCache.cs b/src/Snap.Hutao/Snap.Hutao/Core/Caching/ImageCache.cs index 56381866..2b9a6a40 100644 --- a/src/Snap.Hutao/Snap.Hutao/Core/Caching/ImageCache.cs +++ b/src/Snap.Hutao/Snap.Hutao/Core/Caching/ImageCache.cs @@ -1,9 +1,13 @@ // Copyright (c) DGP Studio. All rights reserved. // Licensed under the MIT license. -using Microsoft.UI.Xaml.Media.Imaging; using Snap.Hutao.Core.DependencyInjection.Annotation.HttpClient; +using Snap.Hutao.Core.Logging; +using System.Collections.Immutable; +using System.IO; using System.Net.Http; +using System.Security.Cryptography; +using System.Text; using Windows.Storage; using Windows.Storage.FileProperties; @@ -16,20 +20,159 @@ namespace Snap.Hutao.Core.Caching; [Injection(InjectAs.Singleton, typeof(IImageCache))] [HttpClient(HttpClientConfigration.Default)] [PrimaryHttpMessageHandler(MaxConnectionsPerServer = 16)] -public class ImageCache : CacheBase, IImageCache +[SuppressMessage("", "CA1001")] +public class ImageCache : IImageCache { private const string DateAccessedProperty = "System.DateAccessed"; + private static readonly ImmutableDictionary retryCountToDelay = new Dictionary() + { + [0] = TimeSpan.FromSeconds(4), + [1] = TimeSpan.FromSeconds(16), + [2] = TimeSpan.FromSeconds(64), + [3] = TimeSpan.FromSeconds(4), + [4] = TimeSpan.FromSeconds(16), + [5] = TimeSpan.FromSeconds(64), + }.ToImmutableDictionary(); + private readonly List extendedPropertyNames = new() { DateAccessedProperty }; + private readonly SemaphoreSlim cacheFolderSemaphore = new(1); + private readonly ILogger logger; + + // violate di rule + private readonly HttpClient httpClient; + + private StorageFolder? baseFolder; + private string? cacheFolderName; + private StorageFolder? cacheFolder; + /// /// Initializes a new instance of the class. /// /// 日志器 /// http客户端工厂 public ImageCache(ILogger logger, IHttpClientFactory httpClientFactory) - : base(logger, httpClientFactory.CreateClient(nameof(ImageCache))) { + this.logger = logger; + httpClient = httpClientFactory.CreateClient(nameof(ImageCache)); + + CacheDuration = TimeSpan.FromDays(30); + RetryCount = 3; + } + + /// + /// Gets or sets the life duration of every cache entry. + /// + public TimeSpan CacheDuration { get; } + + /// + /// Gets or sets the number of retries trying to ensure the file is cached. + /// + public uint RetryCount { get; } + + /// + /// Clears all files in the cache + /// + /// awaitable task + public async Task ClearAsync() + { + StorageFolder folder = await GetCacheFolderAsync().ConfigureAwait(false); + IReadOnlyList files = await folder.GetFilesAsync().AsTask().ConfigureAwait(false); + + await RemoveAsync(files).ConfigureAwait(false); + } + + /// + /// Removes cached files that have expired + /// + /// Optional timespan to compute whether file has expired or not. If no value is supplied, is used. + /// awaitable task + public async Task RemoveExpiredAsync(TimeSpan? duration = null) + { + TimeSpan expiryDuration = duration ?? CacheDuration; + + StorageFolder folder = await GetCacheFolderAsync().ConfigureAwait(false); + IReadOnlyList files = await folder.GetFilesAsync().AsTask().ConfigureAwait(false); + + List filesToDelete = new(); + + foreach (StorageFile file in files) + { + if (file == null) + { + continue; + } + + if (await IsFileOutOfDateAsync(file, expiryDuration, false).ConfigureAwait(false)) + { + filesToDelete.Add(file); + } + } + + await RemoveAsync(filesToDelete).ConfigureAwait(false); + } + + /// + /// Removed items based on uri list passed + /// + /// Enumerable uri list + /// awaitable Task + public async Task RemoveAsync(IEnumerable uriForCachedItems) + { + if (uriForCachedItems == null || !uriForCachedItems.Any()) + { + return; + } + + StorageFolder folder = await GetCacheFolderAsync().ConfigureAwait(false); + IReadOnlyList files = await folder.GetFilesAsync().AsTask().ConfigureAwait(false); + + List filesToDelete = new(); + + Dictionary cachedFiles = files.ToDictionary(file => file.Name); + + foreach (Uri uri in uriForCachedItems) + { + string fileName = GetCacheFileName(uri); + if (cachedFiles.TryGetValue(fileName, out StorageFile? file)) + { + filesToDelete.Add(file); + } + } + + await RemoveAsync(filesToDelete).ConfigureAwait(false); + } + + /// + /// Gets the StorageFile containing cached item for given Uri + /// + /// Uri of the item. + /// a StorageFile + public async Task GetFileFromCacheAsync(Uri uri) + { + StorageFolder folder = await GetCacheFolderAsync().ConfigureAwait(false); + + string fileName = GetCacheFileName(uri); + + IStorageItem? item = await folder.TryGetItemAsync(fileName).AsTask().ConfigureAwait(false); + + if (item == null || (await item.GetBasicPropertiesAsync()).Size == 0) + { + StorageFile baseFile = await folder.CreateFileAsync(fileName, CreationCollisionOption.ReplaceExisting).AsTask().ConfigureAwait(false); + await DownloadFileAsync(uri, baseFile).ConfigureAwait(false); + item = await folder.TryGetItemAsync(fileName).AsTask().ConfigureAwait(false); + } + + return Must.NotNull((item as StorageFile)!); + } + + private static string GetCacheFileName(Uri uri) + { + string url = uri.ToString(); + byte[] chars = Encoding.UTF8.GetBytes(url); + byte[] hash = SHA1.HashData(chars); + return System.Convert.ToHexString(hash); } /// @@ -39,7 +182,7 @@ public class ImageCache : CacheBase, IImageCache /// cache duration /// option to mark uninitialized file as expired /// bool indicate whether file has expired or not - protected override async Task IsFileOutOfDateAsync(StorageFile file, TimeSpan duration, bool treatNullFileAsOutOfDate = true) + private async Task IsFileOutOfDateAsync(StorageFile file, TimeSpan duration, bool treatNullFileAsOutOfDate = true) { if (file == null) { @@ -72,4 +215,92 @@ public class ImageCache : CacheBase, IImageCache return properties.Size == 0 || DateTime.Now.Subtract(properties.DateModified.DateTime) > duration; } + + private async Task DownloadFileAsync(Uri uri, StorageFile baseFile) + { + logger.LogInformation(EventIds.FileCaching, "Begin downloading for {uri}", uri); + + int retryCount = 0; + while (retryCount < 6) + { + using (HttpResponseMessage message = await httpClient.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false)) + { + if (message.IsSuccessStatusCode) + { + using (Stream httpStream = await message.Content.ReadAsStreamAsync().ConfigureAwait(false)) + { + using (FileStream fileStream = File.Create(baseFile.Path)) + { + await httpStream.CopyToAsync(fileStream).ConfigureAwait(false); + return; + } + } + } + else + { + retryCount++; + TimeSpan delay = message.Headers.RetryAfter?.Delta ?? retryCountToDelay[retryCount]; + await Task.Delay(delay).ConfigureAwait(false); + } + } + + if (retryCount == 3) + { + uri = new UriBuilder(uri) { Host = "static.hut.ao", }.Uri; + } + } + } + + /// + /// Initializes with default values if user has not initialized explicitly + /// + /// awaitable task + private async Task InitializeInternalAsync() + { + if (cacheFolder != null) + { + return; + } + + using (await cacheFolderSemaphore.EnterAsync().ConfigureAwait(false)) + { + baseFolder ??= ApplicationData.Current.TemporaryFolder; + + if (string.IsNullOrWhiteSpace(cacheFolderName)) + { + cacheFolderName = GetType().Name; + } + + cacheFolder = await baseFolder + .CreateFolderAsync(cacheFolderName, CreationCollisionOption.OpenIfExists) + .AsTask() + .ConfigureAwait(false); + } + } + + private async Task GetCacheFolderAsync() + { + if (cacheFolder == null) + { + await InitializeInternalAsync().ConfigureAwait(false); + } + + return Must.NotNull(cacheFolder!); + } + + private async Task RemoveAsync(IEnumerable files) + { + foreach (StorageFile file in files) + { + try + { + logger.LogInformation(EventIds.CacheRemoveFile, "Removing file {file}", file.Path); + await file.DeleteAsync().AsTask().ConfigureAwait(false); + } + catch + { + logger.LogError(EventIds.CacheException, "Failed to delete file: {file}", file.Path); + } + } + } } \ No newline at end of file