Enhance repository settings and logging features

- Updated `updateLatestReferences` to include more reference types.
- Added `SetAllowsLargeFiles` and `SetAnonymousAccess` options in the `Commands` module.
- Renamed `GetZipFile` to `GetZipFileUri` in `DirectoryVersion.Actor.fs` for clarity.
- Enhanced logging across various modules for better operational insights.
- Updated `GetZipFile` in `DirectoryVersion.Server.fs` to use the new URI method.
- Modified the `Repository` module to support new configuration parameters.
- Improved `uploadDirectoryVersions` to include owner and organization details.
- Updated `processCommand` to handle new command types for repository settings.
- Enhanced `GraceCommand` to support new command handlers for repository configurations.
- Updated `Dto` module to include properties for `AnonymousAccess` and `AllowsLargeFiles`.
- Added new logging strings in the `Text` module for user feedback.
- Updated `Theme` module for improved CLI output visuals.
- Enhanced `Storage` module to support new download parameters.
- Added command handlers in the `Repository` module for granular control.

src/Grace.Actors/Branch.Actor.fs

@@ -144,11 +144,40 @@ module Branch =
             task {
                 let mutable newBranchDto = branchDto
 
-                // Get the latest references and update the dto.
-                let referenceTypes = [| Save; Checkpoint; Commit; Promotion; Rebase |]
+                // Get the enabled reference types. This allows us to limit the ReferenceTypes we search for.
+                let enabledReferenceTypes = List<ReferenceType>()
 
+                if branchDto.PromotionEnabled then
+                    enabledReferenceTypes.Add(ReferenceType.Promotion)
+
+                if branchDto.CommitEnabled then enabledReferenceTypes.Add(ReferenceType.Commit)
+
+                if branchDto.CheckpointEnabled then
+                    enabledReferenceTypes.Add(ReferenceType.Checkpoint)
+
+                if branchDto.SaveEnabled then enabledReferenceTypes.Add(ReferenceType.Save)
+                if branchDto.TagEnabled then enabledReferenceTypes.Add(ReferenceType.Tag)
+
+                if branchDto.ExternalEnabled then
+                    enabledReferenceTypes.Add(ReferenceType.External)
+
+                if branchDto.AutoRebaseEnabled then
+                    enabledReferenceTypes.Add(ReferenceType.Rebase)
+
+                let referenceTypes = enabledReferenceTypes.ToArray()
+
+                // Get the latest references.
                 let! latestReferences = getLatestReferenceByReferenceTypes referenceTypes branchDto.BranchId
 
+                // Get the latest reference of any type.
+                let latestReference =
+                    latestReferences.Values
+                        .OrderByDescending(fun referenceDto -> referenceDto.UpdatedAt)
+                        .FirstOrDefault(ReferenceDto.Default)
+
+                newBranchDto <- { newBranchDto with LatestReference = latestReference }
+
+                // Get the latest reference of each type.
                 for kvp in latestReferences do
                     let referenceDto = kvp.Value
 

src/Grace.Actors/Commands.Actor.fs

@@ -104,6 +104,8 @@ module Commands =
             | SetRepositoryType of repositoryVisibility: RepositoryType
             | SetRepositoryStatus of repositoryStatus: RepositoryStatus
             | SetRecordSaves of recordSaves: bool
+            | SetAllowsLargeFiles of allowsLargeFiles: bool
+            | SetAnonymousAccess of anonymousAccess: bool
             | SetDefaultServerApiVersion of defaultServerApiVersion: string
             | SetDefaultBranchName of defaultBranchName: BranchName
             | SetLogicalDeleteDays of duration: single

src/Grace.Actors/Diff.Actor.fs

@@ -301,7 +301,7 @@ module Diff =
                                         return! scanForDifferences graceIndex2 graceIndex1
                                 }
 
-                            //logToConsole $"In Actor.Populate(), got differences."
+                            //logToConsole $"In Actor.Populate(); got differences."
 
                             let repositoryActorProxy = Repository.CreateActorProxy repositoryId1 correlationId
                             let! repositoryDto = repositoryActorProxy.Get correlationId

src/Grace.Actors/DirectoryVersion.Actor.fs

@@ -32,6 +32,7 @@ open System.IO
 open System.IO.Compression
 open System.Linq
 open System.Threading.Tasks
+open System.Reflection.Metadata
 
 module DirectoryVersion =
 
@@ -103,6 +104,9 @@ module DirectoryVersion =
                             |> Seq.fold
                                 (fun directoryVersionDto directoryVersionEvent -> directoryVersionDto |> updateDto directoryVersionEvent.Event)
                                 DirectoryVersionDto.Default
+
+                        logToConsole
+                            $"In DirectoryVersionActor.OnActivateAsync: directoryVersion.DirectoryVersionId: {directoryVersionDto.DirectoryVersion.DirectoryVersionId}; directoryVersion.RelativePath: {directoryVersionDto.DirectoryVersion.RelativePath}."
                     | None -> ()
 
                     logActorActivation log activateStartTime correlationId actorName this.Id (getActorActivationMessage retrievedEvents)
@@ -145,26 +149,30 @@ module DirectoryVersion =
 
             if String.IsNullOrEmpty(currentCommand) then
                 log.LogInformation(
-                    "{CurrentInstant}: Node: {HostName}; Duration: {duration_ms}ms; CorrelationId: {correlationId}; Finished {ActorName}.{MethodName}; Id: {Id}.",
+                    "{CurrentInstant}: Node: {HostName}; Duration: {duration_ms}ms; CorrelationId: {correlationId}; Finished {ActorName}.{MethodName}; DirectoryVersionId: {DirectoryVersionId}; RelativePath: {RelativePath}; RepositoryId: {RepositoryId}.",
                     getCurrentInstantExtended (),
                     getMachineName,
                     duration_ms,
                     this.correlationId,
                     actorName,
                     context.MethodName,
-                    this.Id
+                    directoryVersionDto.DirectoryVersion.DirectoryVersionId,
+                    directoryVersionDto.DirectoryVersion.RelativePath,
+                    directoryVersionDto.DirectoryVersion.RepositoryId
                 )
             else
                 log.LogInformation(
-                    "{CurrentInstant}: Node: {HostName}; Duration: {duration_ms}ms; CorrelationId: {correlationId}; Finished {ActorName}.{MethodName}; Command: {Command}; Id: {Id}.",
+                    "{CurrentInstant}: Node: {HostName}; Duration: {duration_ms}ms; CorrelationId: {correlationId}; Finished {ActorName}.{MethodName}; Command: {Command}; DirectoryVersionId: {DirectoryVersionId}; RelativePath: {RelativePath}; RepositoryId: {RepositoryId}.",
                     getCurrentInstantExtended (),
                     getMachineName,
                     duration_ms,
                     this.correlationId,
                     actorName,
                     context.MethodName,
                     currentCommand,
-                    this.Id
+                    directoryVersionDto.DirectoryVersion.DirectoryVersionId,
+                    directoryVersionDto.DirectoryVersion.RelativePath,
+                    directoryVersionDto.DirectoryVersion.RepositoryId
                 )
 
             logScope.Dispose()
@@ -314,6 +322,10 @@ module DirectoryVersion =
                     // Update the Dto with the event.
                     directoryVersionDto <- directoryVersionDto |> updateDto directoryVersionEvent.Event
 
+                    logToConsole
+                        $"In ApplyEvent(): directoryVersion.DirectoryVersionId: {directoryVersionDto.DirectoryVersion.DirectoryVersionId}; directoryVersion.RelativePath: {directoryVersionDto.DirectoryVersion.RelativePath}."
+
+
                     // Publish the event to the rest of the world.
                     let graceEvent = Events.GraceEvent.DirectoryVersionEvent directoryVersionEvent
                     do! daprClient.PublishEventAsync(GracePubSubService, GraceEventStreamTopic, graceEvent)
@@ -563,8 +575,9 @@ module DirectoryVersion =
                         return directoryVersionDto.RecursiveSize
                 }
 
-            member this.GetZipFile(correlationId: CorrelationId) : Task<UriWithSharedAccessSignature> =
+            member this.GetZipFileUri(correlationId: CorrelationId) : Task<UriWithSharedAccessSignature> =
                 this.correlationId <- correlationId
+                let directoryVersion = directoryVersionDto.DirectoryVersion
 
                 /// Creates a .zip file containing the file contents of the directory version.
                 let createDirectoryVersionZipFile
@@ -580,76 +593,77 @@ module DirectoryVersion =
                         let tempZipPath = Path.Combine(Path.GetTempPath(), zipFileName)
 
                         logToConsole
-                            $"In createDirectoryZipAsync: directoryVersionId: {directoryVersionId}; zipFileName: {zipFileName}; tempZipPath: {tempZipPath}."
+                            $"In createDirectoryZipAsync: directoryVersionId: {directoryVersionId}; relativePath: {directoryVersion.RelativePath}; zipFileName: {zipFileName}; tempZipPath: {tempZipPath}."
 
                         try
-                            // Step 1: Create the ZIP archive
+                            // Step 1: Create the ZIP archive.
                             use zipToCreate = new FileStream(tempZipPath, FileMode.Create, FileAccess.Write, FileShare.None, (64 * 1024))
                             use archive = new ZipArchive(zipToCreate, ZipArchiveMode.Create)
 
-                            // Step 2: Process Subdirectories
-                            for subdirectoryId in subdirectories do
-                                let subZipBlobName = $"{GraceDirectoryVersionStorageFolderName}/{subdirectoryId}.zip"
-                                let! subZipBlobClient = getAzureBlobClient repositoryDto subZipBlobName correlationId
+                            // Using this lock to ensure that the files are added to the .zip one at a time. ZipArchive is not thread-safe.
+                            let lockObject = new System.Threading.Lock()
 
-                                // Check if we already have a .zip file for this subdirectory
-                                let! exists = subZipBlobClient.ExistsAsync()
+                            // Step 2: Process the subdirectories of the current directory.
+                            for subdirectoryId in subdirectories do
+                                // Call the subdirectory actor to get the .zip file URI, which will create of the .zip file if it doesn't already exist.
+                                let subdirectoryActorProxy = DirectoryVersion.CreateActorProxy subdirectoryId correlationId
+                                let! subDirectoryVersion = subdirectoryActorProxy.Get correlationId
+                                logToConsole $"In createDirectoryZipAsync: Processing directory version: {subDirectoryVersion.RelativePath}."
 
-                                logToConsole
-                                    $"In createDirectoryZipAsync: directoryVersionId: {directoryVersionId}; subZipBlobName: {subZipBlobName}; exists: {exists}."
+                                let! subdirectoryZipFileUri = subdirectoryActorProxy.GetZipFileUri correlationId
 
-                                // If we don't, call the subdirectory actor to create it.
-                                if exists.Value = false then
-                                    let subdirectoryActorProxy = DirectoryVersion.CreateActorProxy subdirectoryId correlationId
-                                    let! subdirectoryZipFileUri = subdirectoryActorProxy.GetZipFile correlationId
-                                    ()
+                                // Get an Azure Blob Client for the .zip file.
+                                let subZipBlobName = $"{GraceDirectoryVersionStorageFolderName}/{subdirectoryId}.zip"
+                                let! subZipBlobClient = getAzureBlobClient repositoryDto subZipBlobName correlationId
 
-                                // Now that we know the .zip file for the subdirectory is in Azure Blob Storage,
-                                //   copy the contents to the new .zip we're creating.
+                                // Copy the contents of the subdirectory's .zip file to the new .zip we're creating.
                                 use! subZipStream = subZipBlobClient.OpenReadAsync()
                                 use subArchive = new ZipArchive(subZipStream, ZipArchiveMode.Read)
 
                                 for entry in subArchive.Entries do
                                     if not (String.IsNullOrEmpty(entry.Name)) then
+                                        // Using CompressionLevel.NoCompression because the files are already GZipped.
+                                        // We're just using .zip as an archive format for already-compressed files.
                                         let newEntry = archive.CreateEntry(entry.FullName, CompressionLevel.NoCompression)
                                         newEntry.Comment <- entry.Comment
                                         use entryStream = entry.Open()
                                         use newEntryStream = newEntry.Open()
                                         do! entryStream.CopyToAsync(newEntryStream)
 
-                            // Step 3: Process File Versions
+                            // Step 3: Process the files in the current directory.
                             for fileVersion in fileVersions do
-                                logToConsole $"In createDirectoryZipAsync: Processing file version: {fileVersion.GetObjectFileName}."
+                                logToConsole
+                                    $"In createDirectoryZipAsync: Processing file version: {Path.Combine(directoryVersion.RelativePath, fileVersion.GetObjectFileName)}."
+
                                 let! fileBlobClient = getAzureBlobClientForFileVersion repositoryDto fileVersion correlationId
-                                let! fileExists = fileBlobClient.ExistsAsync()
+                                let! existsResult = fileBlobClient.ExistsAsync()
 
-                                if fileExists.Value = true then
+                                if existsResult.Value = true then
                                     use! fileStream = fileBlobClient.OpenReadAsync()
                                     let zipEntry = archive.CreateEntry(fileVersion.RelativePath, CompressionLevel.NoCompression)
                                     zipEntry.Comment <- fileVersion.GetObjectFileName
                                     use zipEntryStream = zipEntry.Open()
                                     do! fileStream.CopyToAsync(zipEntryStream)
 
                             // Step 4: Upload the new ZIP to Azure Blob Storage
-                            let! destinationBlobClient = getAzureBlobClient repositoryDto blobName correlationId
-
                             // Dispose all of the streams before uploading
                             archive.Dispose()
-                            //do! zipToCreate.FlushAsync()
-                            //do! zipToCreate.DisposeAsync()
 
                             // Upload the new .zip file to Azure Blob Storage
+                            let! destinationBlobClient = getAzureBlobClient repositoryDto blobName correlationId
                             use uploadStream = File.OpenRead(tempZipPath)
                             let! response = destinationBlobClient.UploadAsync(uploadStream, true)
-                            logToConsole $"In createDirectoryZipAsync: Successfully uploaded {zipFileName} to Azure Blob Storage. Response: {response}."
+
+                            logToConsole
+                                $"In createDirectoryZipAsync: Successfully uploaded {zipFileName} for relative path {directoryVersion.RelativePath} to Azure Blob Storage."
+
                             ()
                         finally
                             // Step 5: Delete the local ZIP file
                             if File.Exists(tempZipPath) then File.Delete(tempZipPath)
                     }
 
                 task {
-                    let directoryVersion = directoryVersionDto.DirectoryVersion
                     let repositoryActorProxy = Repository.CreateActorProxy directoryVersion.RepositoryId correlationId
                     let! repositoryDto = repositoryActorProxy.Get correlationId
 

src/Grace.Actors/Events.Actor.fs

@@ -158,6 +158,8 @@ module Events =
             | StorageContainerNameSet of storageContainerName: StorageContainerName
             | RepositoryTypeSet of repositoryVisibility: RepositoryType
             | RepositoryStatusSet of repositoryStatus: RepositoryStatus
+            | AllowsLargeFilesSet of allowsLargeFiles: bool
+            | AnonymousAccessSet of anonymousAccess: bool
             | RecordSavesSet of recordSaves: bool
             | DefaultServerApiVersionSet of defaultServerApiVersion: string
             | DefaultBranchNameSet of defaultBranchName: BranchName

src/Grace.Actors/Grace.Actors.fsproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 	<PropertyGroup>
 		<TargetFramework>net9.0</TargetFramework>
@@ -46,6 +46,10 @@
 		<Compile Include="DirectoryAppearance.Actor.fs" />
 	</ItemGroup>
 
+	<ItemGroup>
+	  <None Include="Repository.Actor.fs %28ApplyEvent Method%29" />
+	</ItemGroup>
+
 	<ItemGroup>
 		<PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
 		<PackageReference Include="Azure.Storage.Blobs.Batch" Version="12.20.0" />

src/Grace.Actors/Interfaces.Actor.fs

@@ -144,7 +144,7 @@ module Interfaces =
         abstract member GetRecursiveSize: correlationId: CorrelationId -> Task<int64>
 
         /// Creates the Zip file that contains the contents of this directory and all subdirectories, if it doesn't already exist, and returns a Uri with a shared access signature to download the file.
-        abstract member GetZipFile: correlationId: CorrelationId -> Task<UriWithSharedAccessSignature>
+        abstract member GetZipFileUri: correlationId: CorrelationId -> Task<UriWithSharedAccessSignature>
 
         /// Delete the DirectoryVersion and all subdirectories and files.
         abstract member Delete: correlationId: CorrelationId -> Task<GraceResult<string>>

src/Grace.Actors/Repository.Actor.fs

@@ -175,6 +175,8 @@ module Repository =
                 | LogicalDeleted _ -> { currentRepositoryDto with DeletedAt = Some(getCurrentInstant ()) }
                 | PhysicalDeleted -> currentRepositoryDto // Do nothing because it's about to be deleted anyway.
                 | Undeleted -> { currentRepositoryDto with DeletedAt = None; DeleteReason = String.Empty }
+                | AllowsLargeFilesSet allowsLargeFiles -> { currentRepositoryDto with AllowsLargeFiles = allowsLargeFiles }
+                | AnonymousAccessSet anonymousAccess -> { currentRepositoryDto with AnonymousAccess = anonymousAccess }
 
             { newRepositoryDto with UpdatedAt = Some repositoryEvent.Metadata.Timestamp }
 
@@ -597,6 +599,8 @@ module Repository =
                                     | SetStorageContainerName containerName -> return StorageContainerNameSet containerName
                                     | SetRepositoryStatus repositoryStatus -> return RepositoryStatusSet repositoryStatus
                                     | SetRepositoryType repositoryType -> return RepositoryTypeSet repositoryType
+                                    | SetAllowsLargeFiles allowsLargeFiles -> return AllowsLargeFilesSet allowsLargeFiles
+                                    | SetAnonymousAccess anonymousAccess -> return AnonymousAccessSet anonymousAccess
                                     | SetRecordSaves recordSaves -> return RecordSavesSet recordSaves
                                     | SetDefaultServerApiVersion version -> return DefaultServerApiVersionSet version
                                     | SetDefaultBranchName defaultBranchName -> return DefaultBranchNameSet defaultBranchName