How to store documents in azure blob storage #12471

0

oguzhanagir created one year ago

Support Team

Hi @Prasanthtp

You can create a background job to migrate your existing documents from SQL BLOB storage to Azure Blob Storage while maintaining system availability.

First, add the Azure Storage packages to your project

<PackageReference Include="Azure.Storage.Blobs" Version="12.24.0" />
<PackageReference Include="Microsoft.Extensions.Azure" Version="1.11.0" />

Create a service class for handling Azure Blob operations

public interface IBlobStorageService : ITransientDependency
{
    Task<string> UploadBlobAsync(byte[] data, string fileName, string contentType);
    Task<byte[]> DownloadBlobAsync(string blobName);
    Task DeleteBlobAsync(string blobName);
}

public class BlobStorageService : IBlobStorageService
{
    private readonly BlobServiceClient _blobServiceClient;
    private readonly string _containerName;

    public BlobStorageService(IConfiguration configuration)
    {
        //Add the connection string to your appsettings.json file
        _blobServiceClient = new BlobServiceClient(configuration.GetConnectionString("AzureStorage")); 
        _containerName = configuration["Azure:BlobStorage:ContainerName"];
    }

    public async Task<string> UploadBlobAsync(byte[] data, string fileName, string contentType)
    {
        var containerClient = _blobServiceClient.GetBlobContainerClient(_containerName);
        await containerClient.CreateIfNotExistsAsync(PublicAccessType.None);

        var blobName = $"{Guid.NewGuid()}_{fileName}";
        var blobClient = containerClient.GetBlobClient(blobName);

        using var stream = new MemoryStream(data);
        await blobClient.UploadAsync(stream, new BlobHttpHeaders { ContentType = contentType });

        return blobName;
    }

    public async Task<byte[]> DownloadBlobAsync(string blobName)
    {
        var containerClient = _blobServiceClient.GetBlobContainerClient(_containerName);
        var blobClient = containerClient.GetBlobClient(blobName);

        var response = await blobClient.DownloadAsync();
        using var stream = new MemoryStream();
        await response.Value.Content.CopyToAsync(stream);
        return stream.ToArray();
    }

    public async Task DeleteBlobAsync(string blobName)
    {
        var containerClient = _blobServiceClient.GetBlobContainerClient(_containerName);
        var blobClient = containerClient.GetBlobClient(blobName);
        await blobClient.DeleteIfExistsAsync();
    }
}

Modify your document entity to include blob reference instead of binary data

public class Document : FullAuditedEntity<long>
{
    public string FileName { get; set; }
    public string ContentType { get; set; }
    public long FileSize { get; set; }
    
    // Remove: public byte[] Data { get; set; }
    // Add:
    public string BlobName { get; set; } // Reference to Azure Blob
    public bool IsMigrated { get; set; } = false; // Track migration status
}

Create a background job to migrate existing documents

public class DocumentMigrationJob : BackgroundJob<DocumentMigrationJobArgs>, ITransientDependency
{
    private readonly IRepository<Document, long> _documentRepository;
    private readonly IBlobStorageService _blobStorageService;
    private readonly ILogger<DocumentMigrationJob> _logger;

    public DocumentMigrationJob(
        IRepository<Document, long> documentRepository,
        IBlobStorageService blobStorageService,
        ILogger<DocumentMigrationJob> logger)
    {
        _documentRepository = documentRepository;
        _blobStorageService = blobStorageService;
        _logger = logger;
    }

    public override async Task ExecuteAsync(DocumentMigrationJobArgs args)
    {
        var batchSize = 100;
        var skip = 0;

        while (true)
        {
            // Get documents that haven't been migrated yet
            var documents = await _documentRepository.GetAllListAsync(
                d => !d.IsMigrated && d.Data != null, 
                skip, 
                batchSize);

            if (!documents.Any())
                break;

            foreach (var document in documents)
            {
                try
                {
                    // Upload to Azure Blob Storage
                    var blobName = await _blobStorageService.UploadBlobAsync(
                        document.Data, 
                        document.FileName, 
                        document.ContentType);

                    // Update document record
                    document.BlobName = blobName;
                    document.IsMigrated = true;
                    // Optionally clear the Data field to save space
                    // document.Data = null;

                    await _documentRepository.UpdateAsync(document);

                    _logger.LogInformation($"Migrated document {document.Id}: {document.FileName}");
                }
                catch (Exception ex)
                {
                    _logger.LogError(ex, $"Failed to migrate document {document.Id}: {document.FileName}");
                }
            }

            skip += batchSize;
        }
    }
}

public class DocumentMigrationJobArgs
{
    // Add any parameters if needed
}

To trigger the migration, you can create a simple controller method (for admin use) - Optional

[AbpAuthorize] // Add appropriate permissions
public class MigrationController : YourProjectNameControllerBase
{
    private readonly IBackgroundJobManager _backgroundJobManager;

    public MigrationController(IBackgroundJobManager backgroundJobManager)
    {
        _backgroundJobManager = backgroundJobManager;
    }

    [HttpPost]
    public async Task StartDocumentMigration()
    {
        await _backgroundJobManager.EnqueueAsync<DocumentMigrationJob, DocumentMigrationJobArgs>(
            new DocumentMigrationJobArgs());
    }
}

0

Prasanthtp created one year ago

Hi Thanks for the reply.

I don't have any existing documents to be migrated. I am working on a new project. I want my new documents to be stored in Azure Blob Storage instead of the database while I upload using a file uploader.

0

oguzhanagir created one year ago

Support Team

Hi @Prasanthtp

In this case, you don't need a background job. You can use the Abp.BlobStoring.Azure package. You can review this document (Blob Storing Azure) for detailed explanation and integration.

3 Answer(s)