問題描述

在上一篇博文(【Azure 存儲服務(wù)】.NET7.0 示例代碼之上傳大文件到Azure Storage Blob (一)：https://www.cnblogs.com/lulight/p/17061631.html)中，介紹了第一種分片的方式上傳文件。 本文章接著介紹第二種方式，使用 Microsoft.Azure.Storage.DataMovement 庫中的 TransferManager.UploadAsync 通過并發(fā)的方式來上傳大文件。

問題回答

第一步：添加 Microsoft.Azure.Storage.DataMovement

dotnet add package Microsoft.Azure.Storage.DataMovement

第二步：編寫示例代碼

        String storageConnectionString = "xxxxxxxxxxxxxxxxxxx";

        CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
        CloudBlobClient blobclient = account.CreateCloudBlobClient();
        CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123"); await blobcontainer.CreateIfNotExistsAsync(); // 獲取文件路徑
        string sourcePath = @"C:\home\bigfiles0120.zip";
        CloudBlockBlob docBlob = blobcontainer.GetBlockBlobReference("bigfiles-2"); await docBlob.DeleteIfExistsAsync(); // 設(shè)置并發(fā)操作的數(shù)量
        TransferManager.Configurations.ParallelOperations = 64; // 設(shè)置單塊 blob 的大小，它必須在 4MB 到 100MB 之間，并且是 4MB 的倍數(shù)，默認(rèn)情況下是 4MB
        TransferManager.Configurations.BlockSize = 64 * 1024 * 1024; // 設(shè)置傳輸上下文并跟蹤上傳進度
        var context = new SingleTransferContext();
        UploadOptions uploadOptions = new UploadOptions
        {
            DestinationAccessCondition = AccessCondition.GenerateIfExistsCondition()
        };
        context.ProgressHandler = new Progress<TransferStatus>(progress => { //顯示上傳進度
            Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred);
        }); // 使用 Stopwatch 查看上傳所需時間
        var timer = System.Diagnostics.Stopwatch.StartNew(); // 上傳 Blob
 TransferManager.UploadAsync(sourcePath, docBlob, uploadOptions, context, CancellationToken.None).Wait();
        timer.Stop();
        Console.WriteLine("Time (millisecond):" + timer.ElapsedMilliseconds);
        Console.WriteLine("upload success");

第一種分片方式上傳和第二步并發(fā)上傳的代碼執(zhí)行對比：

image

全部代碼

Program.cs

// See https://aka.ms/new-console-template for more information

Console.WriteLine("Hello, World! Start to upload big files...");

//第一種上傳文件方法: Microsoft.WindowsAzure.Storage
Console.WriteLine("第一種上傳文件方法: Microsoft.WindowsAzure.Storage");
await UploadMethodOne.WindowsAzureStorageUpload();

//第二種上傳文件方法: Microsoft.Azure.Storage.DataMovement
Console.WriteLine("第二種上傳文件方法: Microsoft.Azure.Storage.DataMovement");
await UploadMethodTwo.DataMovementUploadFiletoBlob();

Console.WriteLine("End!");

UploadMethodOne.cs

using Microsoft.Azure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.RetryPolicies;

public static class UploadMethodOne
{
    public static async Task WindowsAzureStorageUpload()
    {
        TimeSpan backOffPeriod = TimeSpan.FromSeconds(2);
        int retryCount = 1;
        //設(shè)置請求選項
        BlobRequestOptions requestoptions = new BlobRequestOptions()
        {
            SingleBlobUploadThresholdInBytes = 1024 * 1024 * 10, //10MB
            ParallelOperationThreadCount = 12,
            RetryPolicy = new ExponentialRetry(backOffPeriod, retryCount),
        };

        //String storageConnectionString = System.Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.User);
        //Console.WriteLine("String account string : "+storageConnectionString);
        String storageConnectionString = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
        CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
        CloudBlobClient blobclient = account.CreateCloudBlobClient();
        //設(shè)置客戶端默認(rèn)請求選項
        blobclient.DefaultRequestOptions = requestoptions;
        CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123");


        await blobcontainer.CreateIfNotExistsAsync();
        //文件路徑，文件大小 
        string sourcePath = @"C:\home\bigfiles0120.zip";
        CloudBlockBlob blockblob = blobcontainer.GetBlockBlobReference("bigfiles-1");
        //設(shè)置單個塊 Blob 的大?。ǚ謮K方式）
        blockblob.StreamWriteSizeInBytes = 1024 * 1024 * 5;
        try
        {
            Console.WriteLine("uploading");
            //使用 Stopwatch 查看上傳時間
            var timer = System.Diagnostics.Stopwatch.StartNew();
            using (var filestream = System.IO.File.OpenRead(sourcePath))
            {
                await blockblob.UploadFromStreamAsync(filestream);
            }
            timer.Stop();

            Console.WriteLine(timer.ElapsedMilliseconds);

            Console.WriteLine("Upload Successful, Time:" + timer.ElapsedMilliseconds);
        }
        catch (Exception e)
        {
            Console.WriteLine(e.Message);
        }

    }
}

UploadMethodTwo.cs

using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Blob;
using Microsoft.Azure.Storage.DataMovement;
public static class UploadMethodTwo
{
    public async static Task DataMovementUploadFiletoBlob()
    {
        String storageConnectionString = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";

        CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
        CloudBlobClient blobclient = account.CreateCloudBlobClient();
        CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123");
        await blobcontainer.CreateIfNotExistsAsync();

        // 獲取文件路徑
        string sourcePath = @"C:\home\bigfiles0120.zip";
        CloudBlockBlob docBlob = blobcontainer.GetBlockBlobReference("bigfiles-2");
        await docBlob.DeleteIfExistsAsync();

        // 設(shè)置并發(fā)操作的數(shù)量
        TransferManager.Configurations.ParallelOperations = 64;
        // 設(shè)置單塊 blob 的大小，它必須在 4MB 到 100MB 之間，并且是 4MB 的倍數(shù)，默認(rèn)情況下是 4MB
        TransferManager.Configurations.BlockSize = 64 * 1024 * 1024;
        // 設(shè)置傳輸上下文并跟蹤上傳進度
        var context = new SingleTransferContext();
        UploadOptions uploadOptions = new UploadOptions
        {
            DestinationAccessCondition = AccessCondition.GenerateIfExistsCondition()
        };
        context.ProgressHandler = new Progress<TransferStatus>(progress =>
        {
            //顯示上傳進度
            Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred);
        });
        // 使用 Stopwatch 查看上傳所需時間
        var timer = System.Diagnostics.Stopwatch.StartNew();
        // 上傳 Blob
        TransferManager.UploadAsync(sourcePath, docBlob, uploadOptions, context, CancellationToken.None).Wait();
        timer.Stop();
        Console.WriteLine("Time (millisecond):" + timer.ElapsedMilliseconds);
        Console.WriteLine("upload success");
    }
}

參考資料

上傳大文件到 Azure 存儲塊 Blob：https://docs.azure.cn/zh-cn/articles/azure-operations-guide/storage/aog-storage-blob-howto-upload-big-file-to-storage

當(dāng)在復(fù)雜的環(huán)境中面臨問題，格物之道需：濁而靜之徐清，安以動之徐生。 云中，恰是如此!

分類: 【Azure Developer】, 【Azure 存儲服務(wù)】

標(biāo)簽: Storage Blob Upload, Microsoft.Azure.Storage.DataMovemen, TransferManager.UploadAsync