Skip to content

Commit

Permalink
Finished implementing grace directory-version get-zip-file.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ScottArbeit
ScottArbeit committed Jan 20, 2025
1 parent e4bb9de commit eb742e2
Show file tree
Hide file tree
Showing 25 changed files with 628 additions and 164 deletions.
8 changes: 3 additions & 5 deletions src/Grace.Actors/Diff.Actor.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,11 +314,9 @@ module Diff =
let directory = graceIndex[relativeDirectoryPath]
let fileVersion = directory.Files.First(fun f -> f.RelativePath = relativePath)

match! getUriWithReadSharedAccessSignatureForFileVersion repositoryDto fileVersion correlationId with
| Ok uri ->
let! uncompressedStream = this.getUncompressedStream repositoryDto fileVersion uri correlationId
return Ok(uncompressedStream, fileVersion)
| Error ex -> return Error ex
let! uri = getUriWithReadSharedAccessSignatureForFileVersion repositoryDto fileVersion correlationId
let! uncompressedStream = this.getUncompressedStream repositoryDto fileVersion uri correlationId
return Ok(uncompressedStream, fileVersion)
}

// Process each difference.
Expand Down
86 changes: 47 additions & 39 deletions src/Grace.Actors/DirectoryVersion.Actor.fs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
namespace Grace.Actors

open Azure.Storage.Blobs
open Azure.Storage.Blobs.Models
open Azure.Storage.Blobs.Specialized
open Dapr.Actors
Expand Down Expand Up @@ -27,11 +28,10 @@ open System
open System.Collections.Concurrent
open System.Collections.Generic
open System.Diagnostics
open System.Linq
open System.Threading.Tasks
open OrganizationName
open System.IO
open System.IO.Compression
open System.Linq
open System.Threading.Tasks

module DirectoryVersion =

Expand Down Expand Up @@ -218,7 +218,7 @@ module DirectoryVersion =
)

return Ok()
| ReminderTypes.DeleteCachedDirectoryVersionContents ->
| ReminderTypes.DeleteZipFile ->
// Get values from state.
if not <| String.IsNullOrEmpty reminder.State then
let (deleteReason, correlationId) = deserialize<PhysicalDeletionReminderState> reminder.State
Expand All @@ -230,7 +230,7 @@ module DirectoryVersion =
let! repositoryDto = repositoryActorProxy.Get correlationId

// Delete cached directory version contents for this actor.
let blobName = $"Grace-DirectoryVersionContents/{directoryVersion.DirectoryVersionId}.zip"
let blobName = $"{GraceDirectoryVersionStorageFolderName}/{directoryVersion.DirectoryVersionId}.zip"
let! zipFileBlobClient = getAzureBlobClient repositoryDto blobName correlationId

let! deleted = zipFileBlobClient.DeleteIfExistsAsync()
Expand Down Expand Up @@ -563,15 +563,17 @@ module DirectoryVersion =
return directoryVersionDto.RecursiveSize
}

member this.CreateZipFile(correlationId: CorrelationId) : Task<GraceResult<string>> =
member this.GetZipFile(correlationId: CorrelationId) : Task<UriWithSharedAccessSignature> =
this.correlationId <- correlationId

let createDirectoryZipAsync
/// Creates a .zip file containing the file contents of the directory version.
let createDirectoryVersionZipFile
(repositoryDto: RepositoryDto)
(blobName: string)
(directoryVersionId: DirectoryVersionId)
(subdirectories: List<DirectoryVersionId>)
(fileVersions: IEnumerable<FileVersion>)
: Task =
=

task {
let zipFileName = $"{directoryVersionId}.zip"
Expand All @@ -587,7 +589,7 @@ module DirectoryVersion =

// Step 2: Process Subdirectories
for subdirectoryId in subdirectories do
let subZipBlobName = $"Grace-DirectoryVersionContents/{subdirectoryId}.zip"
let subZipBlobName = $"{GraceDirectoryVersionStorageFolderName}/{subdirectoryId}.zip"
let! subZipBlobClient = getAzureBlobClient repositoryDto subZipBlobName correlationId

// Check if we already have a .zip file for this subdirectory
Expand All @@ -599,10 +601,8 @@ module DirectoryVersion =
// If we don't, call the subdirectory actor to create it.
if exists.Value = false then
let subdirectoryActorProxy = DirectoryVersion.CreateActorProxy subdirectoryId correlationId

match! subdirectoryActorProxy.CreateZipFile correlationId with
| Ok graceReturnValue -> logToConsole $"Successfully created subdirectory .zip file: {subdirectoryId}.zip"
| Error graceError -> logToConsole $"Error creating subdirectory .zip file: {subdirectoryId}.zip. Error: {graceError}"
let! subdirectoryZipFileUri = subdirectoryActorProxy.GetZipFile correlationId
()

// Now that we know the .zip file for the subdirectory is in Azure Blob Storage,
// copy the contents to the new .zip we're creating.
Expand All @@ -619,7 +619,7 @@ module DirectoryVersion =

// Step 3: Process File Versions
for fileVersion in fileVersions do
logToConsole $"In createDirectoryZipAsync: Processing file version: {fileVersion.RelativePath}."
logToConsole $"In createDirectoryZipAsync: Processing file version: {fileVersion.GetObjectFileName}."
let! fileBlobClient = getAzureBlobClientForFileVersion repositoryDto fileVersion correlationId
let! fileExists = fileBlobClient.ExistsAsync()

Expand All @@ -631,8 +631,7 @@ module DirectoryVersion =
do! fileStream.CopyToAsync(zipEntryStream)

// Step 4: Upload the new ZIP to Azure Blob Storage
let destinationBlobName = $"Grace-DirectoryVersionContents/{zipFileName}"
let! destinationBlobClient = getAzureBlobClient repositoryDto destinationBlobName correlationId
let! destinationBlobClient = getAzureBlobClient repositoryDto blobName correlationId

// Dispose all of the streams before uploading
archive.Dispose()
Expand All @@ -650,30 +649,39 @@ module DirectoryVersion =
}

task {
try
let directoryVersion = directoryVersionDto.DirectoryVersion
let repositoryActorProxy = Repository.CreateActorProxy directoryVersion.RepositoryId correlationId
let! repositoryDto = repositoryActorProxy.Get correlationId

let blobName = $"Grace-DirectoryVersionContents/{directoryVersion.DirectoryVersionId}.zip"
let! zipFileBlobClient = getAzureBlobClient repositoryDto blobName correlationId

let! zipFileExists = zipFileBlobClient.ExistsAsync()
let directoryVersion = directoryVersionDto.DirectoryVersion
let repositoryActorProxy = Repository.CreateActorProxy directoryVersion.RepositoryId correlationId
let! repositoryDto = repositoryActorProxy.Get correlationId

if zipFileExists.Value = true then
return Ok(GraceReturnValue.Create "Zip file already exists." correlationId)
else
do! createDirectoryZipAsync repositoryDto directoryVersion.DirectoryVersionId directoryVersion.Directories directoryVersion.Files
let deletionReminderState = (getDiscriminatedUnionCaseName ReminderTypes.DeleteCachedDirectoryVersionContents, correlationId)
let blobName = $"{GraceDirectoryVersionStorageFolderName}/{directoryVersion.DirectoryVersionId}.zip"
let! zipFileBlobClient = getAzureBlobClient repositoryDto blobName correlationId

do!
(this :> IGraceReminder).ScheduleReminderAsync
ReminderTypes.DeleteCachedDirectoryVersionContents
(Duration.FromDays(float repositoryDto.DirectoryVersionCacheDays))
(serialize deletionReminderState)
correlationId
let! zipFileExists = zipFileBlobClient.ExistsAsync()

return Ok(GraceReturnValue.Create "Zip file created succeeded." correlationId)
with ex ->
return Error(GraceError.Create $"{ExceptionResponse.Create ex}" correlationId)
if zipFileExists.Value = true then
// We already have this .zip file, so just return the URI with SAS.
let! uriWithSas = getUriWithReadSharedAccessSignature repositoryDto blobName correlationId
return uriWithSas
else
// We don't have the .zip file saved, so let's create it.
do!
createDirectoryVersionZipFile
repositoryDto
blobName
directoryVersion.DirectoryVersionId
directoryVersion.Directories
directoryVersion.Files

// Schedule a reminder to delete the .zip file after the cache days have passed.
let deletionReminderState = (getDiscriminatedUnionCaseName DeleteZipFile, correlationId)

do!
(this :> IGraceReminder).ScheduleReminderAsync
DeleteZipFile
(Duration.FromDays(float repositoryDto.DirectoryVersionCacheDays))
(serialize deletionReminderState)
correlationId

let! uriWithSas = getUriWithReadSharedAccessSignature repositoryDto blobName correlationId
return uriWithSas
}
4 changes: 2 additions & 2 deletions src/Grace.Actors/Interfaces.Actor.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ module Interfaces =
/// Returns the total size of files contained in this directory and all subdirectories.
abstract member GetRecursiveSize: correlationId: CorrelationId -> Task<int64>

/// Creates the Zip file that contains the contents of this directory and all subdirectories, if it doesn't already exist.
abstract member CreateZipFile: correlationId: CorrelationId -> Task<GraceResult<string>>
/// Creates the Zip file that contains the contents of this directory and all subdirectories, if it doesn't already exist, and returns a Uri with a shared access signature to download the file.
abstract member GetZipFile: correlationId: CorrelationId -> Task<UriWithSharedAccessSignature>

/// Delete the DirectoryVersion and all subdirectories and files.
abstract member Delete: correlationId: CorrelationId -> Task<GraceResult<string>>
Expand Down
48 changes: 28 additions & 20 deletions src/Grace.Actors/Services.Actor.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ module Services =
graceError

/// Gets an Azure Blob Storage container client for the container that holds the object files for the given repository.
let getContainerClient (repositoryDto: RepositoryDto) =
let getContainerClient (repositoryDto: RepositoryDto) correlationId =
task {
let containerName = $"{repositoryDto.RepositoryId}"
let key = $"Con:{repositoryDto.StorageAccountName}-{containerName}"
Expand All @@ -124,15 +124,23 @@ module Services =
cacheEntry.AbsoluteExpiration <- DateTimeOffset.UtcNow.Add(TimeSpan.FromMinutes(10.0))

let blobContainerClient = BlobContainerClient(azureStorageConnectionString, containerName)

// Make sure the container exists before returning the client.
let metadata = Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) :> IDictionary<string, string>
metadata[nameof (OwnerId)] <- $"{repositoryDto.OwnerId}"
metadata[nameof (OrganizationId)] <- $"{repositoryDto.OrganizationId}"
metadata[nameof (RepositoryId)] <- $"{repositoryDto.RepositoryId}"

let! azureResponse =
blobContainerClient.CreateIfNotExistsAsync(publicAccessType = Models.PublicAccessType.None, metadata = metadata)
let! exists = blobContainerClient.ExistsAsync()

if exists.Value = false then
let ownerActorProxy = Owner.CreateActorProxy repositoryDto.OwnerId CorrelationId.Empty
let! ownerDto = ownerActorProxy.Get correlationId
let organizationActorProxy = Organization.CreateActorProxy repositoryDto.OrganizationId CorrelationId.Empty
let! organizationDto = organizationActorProxy.Get correlationId
let metadata = Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) :> IDictionary<string, string>
metadata[nameof (OwnerId)] <- $"{repositoryDto.OwnerId}"
metadata[nameof (OwnerName)] <- $"{ownerDto.OwnerName}"
metadata[nameof (OrganizationId)] <- $"{repositoryDto.OrganizationId}"
metadata[nameof (OrganizationName)] <- $"{organizationDto.OrganizationName}"
metadata[nameof (RepositoryId)] <- $"{repositoryDto.RepositoryId}"
metadata[nameof (RepositoryName)] <- $"{repositoryDto.RepositoryName}"

let! azureResponse = blobContainerClient.CreateAsync(publicAccessType = Models.PublicAccessType.None, metadata = metadata)
()

return blobContainerClient
}
Expand All @@ -145,7 +153,7 @@ module Services =
let getAzureBlobClient (repositoryDto: RepositoryDto) (blobName: string) (correlationId: CorrelationId) =
task {
//logToConsole $"* In getAzureBlobClient; repositoryId: {repositoryDto.RepositoryId}; fileVersion: {fileVersion.RelativePath}."
let! containerClient = getContainerClient repositoryDto
let! containerClient = getContainerClient repositoryDto correlationId

return containerClient.GetBlobClient(blobName)
}
Expand All @@ -160,7 +168,7 @@ module Services =
let private createAzureBlobSasUri (repositoryDto: RepositoryDto) (blobName: string) (permission: BlobSasPermissions) (correlationId: CorrelationId) =
task {
//logToConsole $"In createAzureBlobSasUri; fileVersion.RelativePath: {fileVersion.RelativePath}."
let! blobContainerClient = getContainerClient repositoryDto
let! blobContainerClient = getContainerClient repositoryDto correlationId

let blobSasBuilder =
BlobSasBuilder(
Expand All @@ -184,10 +192,10 @@ module Services =
| AzureBlobStorage ->
let permissions = (BlobSasPermissions.Read ||| BlobSasPermissions.List) // These are the minimum permissions needed to read a file.
let! sas = createAzureBlobSasUri repositoryDto blobName permissions correlationId
return Ok sas
| AWSS3 -> return Error(NotImplementedException("AWS S3 storage type is not implemented."))
| GoogleCloudStorage -> return Error(NotImplementedException("Google Cloud storage type is not implemented."))
| ObjectStorageProvider.Unknown -> return Error(NotImplementedException("Unknown storage type."))
return sas
| AWSS3 -> return UriWithSharedAccessSignature(String.Empty)
| GoogleCloudStorage -> return UriWithSharedAccessSignature(String.Empty)
| ObjectStorageProvider.Unknown -> return UriWithSharedAccessSignature(String.Empty)
}

/// Gets a full Uri, including shared access signature, for reading from the object storage provider.
Expand All @@ -201,7 +209,7 @@ module Services =
let getUriWithWriteSharedAccessSignature (repositoryDto: RepositoryDto) (blobName: string) (correlationId: CorrelationId) =
task {
match repositoryDto.ObjectStorageProvider with
| AWSS3 -> return Error(NotImplementedException("AWS S3 storage type is not implemented."))
| AWSS3 -> return UriWithSharedAccessSignature(String.Empty)
| AzureBlobStorage ->
// Adding read permission to allow for calls to .ExistsAsync().
let permissions =
Expand All @@ -213,9 +221,9 @@ module Services =

let! sas = createAzureBlobSasUri repositoryDto blobName permissions correlationId

return Ok sas
| GoogleCloudStorage -> return Error(NotImplementedException("Google Cloud storage type is not implemented."))
| ObjectStorageProvider.Unknown -> return Error(NotImplementedException("Unknown storage type."))
return sas
| GoogleCloudStorage -> return UriWithSharedAccessSignature(String.Empty)
| ObjectStorageProvider.Unknown -> return UriWithSharedAccessSignature(String.Empty)
}

/// Gets a full Uri, including shared access signature, for writing from the object storage provider.
Expand Down
Loading

0 comments on commit eb742e2

Please sign in to comment.