问题描述
在上一篇博文(【Azure 存储服务】.NET7.0 示例代码之上传大文件到Azure Storage Blob (一)：https://www.cnblogs.com/lulight/p/17061631.html)中，介绍了第一种分片的方式上传文件。 本文章接着介绍第二种方式，使用 Microsoft.Azure.Storage.DataMovement 库中的 TransferManager.UploadAsync 通过并发的方式来上传大文件。
问题回答
第一步：添加 Microsoft.Azure.Storage.DataMovement
dotnet add package Microsoft.Azure.Storage.DataMovement
第二步：编写示例代码
String storageConnectionString = "xxxxxxxxxxxxxxxxxxx"; CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString); CloudBlobClient blobclient = account.CreateCloudBlobClient(); CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123"); await blobcontainer.CreateIfNotExistsAsync(); // 获取文件路径 string sourcePath = @"C:\home\bigfiles0120.zip"; CloudBlockBlob docBlob = blobcontainer.GetBlockBlobReference("bigfiles-2"); await docBlob.DeleteIfExistsAsync(); // 设置并发操作的数量 TransferManager.Configurations.ParallelOperations = 64; // 设置单块 blob 的大小，它必须在 4MB 到 100MB 之间，并且是 4MB 的倍数，默认情况下是 4MB TransferManager.Configurations.BlockSize = 64 * 1024 * 1024; // 设置传输上下文并跟踪上传进度 var context = new SingleTransferContext(); UploadOptions uploadOptions = new UploadOptions { DestinationAccessCondition = AccessCondition.GenerateIfExistsCondition() }; context.ProgressHandler = new Progress<TransferStatus>(progress => { //显示上传进度 Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred); }); // 使用 Stopwatch 查看上传所需时间 var timer = System.Diagnostics.Stopwatch.StartNew(); // 上传 Blob TransferManager.UploadAsync(sourcePath, docBlob, uploadOptions, context, CancellationToken.None).Wait(); timer.Stop(); Console.WriteLine("Time (millisecond):" + timer.ElapsedMilliseconds); Console.WriteLine("upload success");
第一种分片方式上传和第二步并发上传的代码执行对比：
全部代码
Program.cs
// See https://aka.ms/new-console-template for more information Console.WriteLine("Hello, World! Start to upload big files..."); //第一种上传文件方法: Microsoft.WindowsAzure.Storage Console.WriteLine("第一种上传文件方法: Microsoft.WindowsAzure.Storage"); await UploadMethodOne.WindowsAzureStorageUpload(); //第二种上传文件方法: Microsoft.Azure.Storage.DataMovement Console.WriteLine("第二种上传文件方法: Microsoft.Azure.Storage.DataMovement"); await UploadMethodTwo.DataMovementUploadFiletoBlob(); Console.WriteLine("End!");
UploadMethodOne.cs
using Microsoft.Azure; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; public static class UploadMethodOne { public static async Task WindowsAzureStorageUpload() { TimeSpan backOffPeriod = TimeSpan.FromSeconds(2); int retryCount = 1; //设置请求选项 BlobRequestOptions requestoptions = new BlobRequestOptions() { SingleBlobUploadThresholdInBytes = 1024 * 1024 * 10, //10MB ParallelOperationThreadCount = 12, RetryPolicy = new ExponentialRetry(backOffPeriod, retryCount), }; //String storageConnectionString = System.Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.User); //Console.WriteLine("String account string : "+storageConnectionString); String storageConnectionString = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString); CloudBlobClient blobclient = account.CreateCloudBlobClient(); //设置客户端默认请求选项 blobclient.DefaultRequestOptions = requestoptions; CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123"); await blobcontainer.CreateIfNotExistsAsync(); //文件路径，文件大小 string sourcePath = @"C:\home\bigfiles0120.zip"; CloudBlockBlob blockblob = blobcontainer.GetBlockBlobReference("bigfiles-1"); //设置单个块 Blob 的大小（分块方式） blockblob.StreamWriteSizeInBytes = 1024 * 1024 * 5; try { Console.WriteLine("uploading"); //使用 Stopwatch 查看上传时间 var timer = System.Diagnostics.Stopwatch.StartNew(); using (var filestream = System.IO.File.OpenRead(sourcePath)) { await blockblob.UploadFromStreamAsync(filestream); } timer.Stop(); Console.WriteLine(timer.ElapsedMilliseconds); Console.WriteLine("Upload Successful, Time:" + timer.ElapsedMilliseconds); } catch (Exception e) { Console.WriteLine(e.Message); } } }
UploadMethodTwo.cs
using Microsoft.Azure.Storage; using Microsoft.Azure.Storage.Blob; using Microsoft.Azure.Storage.DataMovement; public static class UploadMethodTwo { public async static Task DataMovementUploadFiletoBlob() { String storageConnectionString = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString); CloudBlobClient blobclient = account.CreateCloudBlobClient(); CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123"); await blobcontainer.CreateIfNotExistsAsync(); // 获取文件路径 string sourcePath = @"C:\home\bigfiles0120.zip"; CloudBlockBlob docBlob = blobcontainer.GetBlockBlobReference("bigfiles-2"); await docBlob.DeleteIfExistsAsync(); // 设置并发操作的数量 TransferManager.Configurations.ParallelOperations = 64; // 设置单块 blob 的大小，它必须在 4MB 到 100MB 之间，并且是 4MB 的倍数，默认情况下是 4MB TransferManager.Configurations.BlockSize = 64 * 1024 * 1024; // 设置传输上下文并跟踪上传进度 var context = new SingleTransferContext(); UploadOptions uploadOptions = new UploadOptions { DestinationAccessCondition = AccessCondition.GenerateIfExistsCondition() }; context.ProgressHandler = new Progress<TransferStatus>(progress => { //显示上传进度 Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred); }); // 使用 Stopwatch 查看上传所需时间 var timer = System.Diagnostics.Stopwatch.StartNew(); // 上传 Blob TransferManager.UploadAsync(sourcePath, docBlob, uploadOptions, context, CancellationToken.None).Wait(); timer.Stop(); Console.WriteLine("Time (millisecond):" + timer.ElapsedMilliseconds); Console.WriteLine("upload success"); } }
参考资料
上传大文件到 Azure 存储块 Blob：https://docs.azure.cn/zh-cn/articles/azure-operations-guide/storage/aog-storage-blob-howto-upload-big-file-to-storage
