diff --git a/src/Grace.Actors/Constants.Actor.fs b/src/Grace.Actors/Constants.Actor.fs index 66b9f3f..de38c9a 100644 --- a/src/Grace.Actors/Constants.Actor.fs +++ b/src/Grace.Actors/Constants.Actor.fs @@ -151,7 +151,7 @@ module Constants = #if DEBUG let DefaultPhysicalDeletionReminderDuration = Duration.FromSeconds(300.0) #else - let DefaultPhysicalDeletionReminderTime = Duration.FromDays(7.0) + let DefaultPhysicalDeletionReminderDuration = Duration.FromDays(7.0) #endif /// The time to wait between logical and physical deletion of an actor's state, as a TimeSpan. diff --git a/src/Grace.Actors/Diff.Actor.fs b/src/Grace.Actors/Diff.Actor.fs index 7f82294..5c69cbc 100644 --- a/src/Grace.Actors/Diff.Actor.fs +++ b/src/Grace.Actors/Diff.Actor.fs @@ -129,14 +129,10 @@ module Diff = } /// Gets a Stream from object storage for a specific FileVersion, using a generated Uri. - member private this.getFileStream (fileVersion: FileVersion) (url: UriWithSharedAccessSignature) correlationId = + member private this.getFileStream (repositoryDto: RepositoryDto) (fileVersion: FileVersion) (url: UriWithSharedAccessSignature) correlationId = task { this.correlationId <- correlationId - let repositoryActorId = Repository.GetActorId(fileVersion.RepositoryId) - - let repositoryActorProxy = - actorProxyFactory.CreateActorProxyWithCorrelationId(repositoryActorId, ActorName.Repository, correlationId) - + let repositoryActorProxy = Repository.CreateActorProxy repositoryDto.RepositoryId correlationId let! objectStorageProvider = repositoryActorProxy.GetObjectStorageProvider correlationId match objectStorageProvider with @@ -312,7 +308,7 @@ module Diff = let directory = graceIndex[relativeDirectoryPath] let fileVersion = directory.Files.First(fun f -> f.RelativePath = relativePath) let! uri = getReadSharedAccessSignature repositoryDto fileVersion correlationId - let! stream = this.getFileStream fileVersion (Result.get uri) correlationId + let! stream = this.getFileStream repositoryDto fileVersion (Result.get uri) correlationId return (stream, fileVersion) } diff --git a/src/Grace.Actors/FileAppearance.Actor.fs b/src/Grace.Actors/FileAppearance.Actor.fs index 538b93b..2e5d6ac 100644 --- a/src/Grace.Actors/FileAppearance.Actor.fs +++ b/src/Grace.Actors/FileAppearance.Actor.fs @@ -19,7 +19,6 @@ open System.Threading.Tasks module FileAppearance = - let GetActorId (fileVersion: FileVersion) = ActorId($"{fileVersion.RepositoryId}*{fileVersion.RelativePath}*{fileVersion.Sha256Hash}") let actorName = ActorName.FileAppearance let log = loggerFactory.CreateLogger("FileAppearance.Actor") diff --git a/src/Grace.Actors/Services.Actor.fs b/src/Grace.Actors/Services.Actor.fs index 612ca21..af384b5 100644 --- a/src/Grace.Actors/Services.Actor.fs +++ b/src/Grace.Actors/Services.Actor.fs @@ -1120,7 +1120,7 @@ module Services = /// Deletes all documents from CosmosDb. /// /// **** This method is implemented only in Debug configuration. It is a no-op in Release configuration. **** - let deleteAllFromCosmosDB () = + let deleteAllFromCosmosDBThatMatch (queryDefinition: QueryDefinition) = task { #if DEBUG let failed = List() @@ -1130,10 +1130,7 @@ module Services = let parallelOptions = ParallelOptions(MaxDegreeOfParallelism = 3) let itemRequestOptions = - ItemRequestOptions(AddRequestHeaders = fun headers -> headers.Add(Constants.CorrelationIdHeaderKey, "Deleting all records from CosmosDB")) - - let queryDefinition = QueryDefinition("""SELECT c.id, c.partitionKey FROM c ORDER BY c.partitionKey""") - //let queryDefinition = QueryDefinition("""SELECT c.id, c.partitionKey FROM c WHERE ENDSWITH(c.id, "||Rmd") ORDER BY c.partitionKey""") + ItemRequestOptions(AddRequestHeaders = fun headers -> headers.Add(Constants.CorrelationIdHeaderKey, "deleteAllFromCosmosDBThatMatch")) let mutable totalRecordsDeleted = 0 let overallStartTime = getCurrentInstant () @@ -1178,7 +1175,7 @@ module Services = let overallRps = float totalRecordsDeleted / overall_duration_s logToConsole - $"In Services.deleteAllFromCosmosDB(): batch duration (s): {duration_s:F3}; batch requests/second: {rps:F3}; failed.Count: {failed.Count}; totalRequestCharge: {float totalRequestCharge / 1000.0:F2}; totalRecordsDeleted: {totalRecordsDeleted}; overall duration (m): {overall_duration_s / 60.0:F3}; overall requests/second: {overallRps:F3}." + $"In Services.deleteAllFromCosmosDBThatMatch(): batch duration (s): {duration_s:F3}; batch requests/second: {rps:F3}; failed.Count: {failed.Count}; totalRequestCharge: {float totalRequestCharge / 1000.0:F2}; totalRecordsDeleted: {totalRecordsDeleted}; overall duration (m): {overall_duration_s / 60.0:F3}; overall requests/second: {overallRps:F3}." return failed with ex -> @@ -1189,6 +1186,32 @@ module Services = #endif } + /// Deletes all documents from CosmosDB. + /// + /// **** This method is implemented only in Debug configuration. It is a no-op in Release configuration. **** + let deleteAllFromCosmosDb () = + task { +#if DEBUG + let queryDefinition = QueryDefinition("""SELECT c.id, c.partitionKey FROM c ORDER BY c.partitionKey""") + return! deleteAllFromCosmosDBThatMatch queryDefinition +#else + return List([ "Not implemented" ]) +#endif + } + + /// Deletes all Reminders from CosmosDB. + /// + /// **** This method is implemented only in Debug configuration. It is a no-op in Release configuration. **** + let deleteAllRemindersFromCosmosDb () = + task { +#if DEBUG + let queryDefinition = QueryDefinition("""SELECT c.id, c.partitionKey FROM c WHERE ENDSWITH(c.id, "||Rmd") ORDER BY c.partitionKey""") + return! deleteAllFromCosmosDBThatMatch queryDefinition +#else + return List([ "Not implemented" ]) +#endif + } + /// Gets a list of references of a given ReferenceType for a branch. let getReferencesByType (referenceType: ReferenceType) (branchId: BranchId) (maxCount: int) (correlationId: CorrelationId) = task { @@ -1372,7 +1395,7 @@ module Services = } /// Gets the latest reference for a given ReferenceType in a branch. - let getLatestReferenceByType referenceType (branchId: BranchId) = + let getLatestReferenceByType (referenceType: ReferenceType) (branchId: BranchId) = task { match actorStateStorageProvider with | Unknown -> return None @@ -1464,7 +1487,6 @@ module Services = AND event.Event.created.RepositoryId = @repositoryId AND event.Event.created.Class = @class""" ) - //let queryDefinition = QueryDefinition("""SELECT TOP 1 c["value"] FROM c WHERE c["value"].RepositoryId = @repositoryId AND STARTSWITH(c["value"].Sha256Hash, @sha256Hash, true) AND c["value"].Class = @class""") .WithParameter("@sha256Hash", sha256Hash) .WithParameter("@repositoryId", repositoryId) .WithParameter("@class", nameof (DirectoryVersion)) diff --git a/src/Grace.Aspire.AppHost/Program.cs b/src/Grace.Aspire.AppHost/Program.cs index 90ac9a6..81cd912 100644 --- a/src/Grace.Aspire.AppHost/Program.cs +++ b/src/Grace.Aspire.AppHost/Program.cs @@ -75,7 +75,7 @@ string daprConfigurationPath() // Add local containers as services for Dapr. var zipkin = builder.AddContainer("zipkin", "openzipkin/zipkin", "latest"); - zipkin.WithEndpoint(hostPort: 9411, name: "zipkin-http", scheme: "http"); + zipkin.WithEndpoint(port: 9411, name: "zipkin-http", scheme: "http"); //var prometheus = builder.AddContainer("prometheus", "prom/prometheus", "latest") // .WithBindMount("../prometheus", "/etc/prometheus", isReadOnly: true); diff --git a/src/Grace.CLI/Command/Branch.CLI.fs b/src/Grace.CLI/Command/Branch.CLI.fs index 039fb54..442f614 100644 --- a/src/Grace.CLI/Command/Branch.CLI.fs +++ b/src/Grace.CLI/Command/Branch.CLI.fs @@ -35,6 +35,7 @@ open System.Security.Cryptography open System.Threading.Tasks open System.Text open System.Text.Json +open Grace.Shared.Parameters module Branch = open Grace.Shared.Validation.Common.Input @@ -798,6 +799,7 @@ module Branch = if parseResult |> verbose then printParseResult parseResult let validateIncomingParameters = CommonValidations parseResult parameters + let repositoryId = RepositoryId.Parse(parameters.RepositoryId) match validateIncomingParameters with | Ok _ -> @@ -884,7 +886,22 @@ module Branch = let mutable lastFileUploadInstant = newGraceStatus.LastSuccessfulFileUpload if newFileVersions.Count() > 0 then - match! uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) with + let getUploadMetadataForFilesParameters = + Storage.GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + match! uploadFilesToObjectStorage getUploadMetadataForFilesParameters with | Ok returnValue -> () //logToAnsiConsole Colors.Verbose $"Uploaded all files to object storage." | Error error -> logToAnsiConsole Colors.Error $"Error uploading files to object storage: {error.Error}" @@ -970,7 +987,22 @@ module Branch = .First(fun dv -> dv.Files.Exists(fun file -> file.RelativePath = relativePath)) .Files.First(fun file -> file.RelativePath = relativePath)) - let! uploadResult = uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) + let getUploadMetadataForFilesParameters = + Storage.GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + let! uploadResult = uploadFilesToObjectStorage getUploadMetadataForFilesParameters let saveParameters = SaveDirectoryVersionsParameters() saveParameters.DirectoryVersions <- newDirectoryVersions.Select(fun dv -> dv.ToDirectoryVersion).ToList() @@ -1926,6 +1958,7 @@ module Branch = ) = task { t |> startProgressTask showOutput + let repositoryId = RepositoryId.Parse(parameters.RepositoryId) if currentBranch.SaveEnabled && newDirectoryVersions.Any() then let updatedRelativePaths = @@ -1954,7 +1987,22 @@ module Branch = Colors.Verbose $"Uploading {newFileVersions.Count()} file(s) from {newDirectoryVersions.Count} new directory version(s) to object storage." - match! uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) with + let getUploadMetadataForFilesParameters = + Storage.GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + match! uploadFilesToObjectStorage getUploadMetadataForFilesParameters with | Ok returnValue -> t |> setProgressTaskValue showOutput 100.0 return Ok(showOutput, parseResult, parameters, currentBranch, newDirectoryVersions) @@ -2369,6 +2417,7 @@ module Branch = let mutable rootDirectoryId = DirectoryVersionId.Empty let mutable rootDirectorySha256Hash = Sha256Hash String.Empty let mutable previousDirectoryIds: HashSet = null + let repositoryId = RepositoryId.Parse(parameters.RepositoryId) if parseResult |> hasOutput then return! @@ -2461,7 +2510,22 @@ module Branch = .First(fun dv -> dv.Files.Exists(fun file -> file.RelativePath = relativePath)) .Files.First(fun file -> file.RelativePath = relativePath)) - let! uploadResult = uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) + let getUploadMetadataForFilesParameters = + Storage.GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + let! uploadResult = uploadFilesToObjectStorage getUploadMetadataForFilesParameters t3.Value <- 100.0 @@ -2688,7 +2752,22 @@ module Branch = .First(fun dv -> dv.Files.Exists(fun file -> file.RelativePath = relativePath)) .Files.First(fun file -> file.RelativePath = relativePath)) - let! uploadResult = uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) + let getUploadMetadataForFilesParameters = + Storage.GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + let! uploadResult = uploadFilesToObjectStorage getUploadMetadataForFilesParameters let saveParameters = SaveDirectoryVersionsParameters() diff --git a/src/Grace.CLI/Command/Diff.CLI.fs b/src/Grace.CLI/Command/Diff.CLI.fs index 9b5f101..7c56968 100644 --- a/src/Grace.CLI/Command/Diff.CLI.fs +++ b/src/Grace.CLI/Command/Diff.CLI.fs @@ -32,6 +32,7 @@ open System.Threading.Tasks open Spectre.Console open Spectre.Console open Spectre.Console.Rendering +open Grace.Shared.Parameters.Storage module Diff = @@ -346,6 +347,7 @@ module Diff = let mutable rootDirectoryId = DirectoryVersionId.Empty let mutable rootDirectorySha256Hash = Sha256Hash String.Empty let mutable previousDirectoryIds: HashSet = null + let repositoryId = RepositoryId.Parse(parameters.RepositoryId) // Check for latest commit and latest root directory version from grace watch. If it's running, we know GraceStatus is up-to-date. match! getGraceWatchStatus () with @@ -380,7 +382,22 @@ module Diff = t3.StartTask() - match! uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) with + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + match! uploadFilesToObjectStorage getUploadMetadataForFilesParameters with | Ok returnValue -> () | Error error -> logToAnsiConsole Colors.Error $"Failed to upload changed files to object storage. {error}" @@ -618,6 +635,7 @@ module Diff = let mutable rootDirectoryId = DirectoryVersionId.Empty let mutable rootDirectorySha256Hash = Sha256Hash String.Empty let mutable previousDirectoryIds: HashSet = null + let repositoryId = RepositoryId.Parse(parameters.RepositoryId) // Check for latest commit and latest root directory version from grace watch. If it's running, we know GraceStatus is up-to-date. match! getGraceWatchStatus () with @@ -653,7 +671,22 @@ module Diff = t3.StartTask() - match! uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) with + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + match! uploadFilesToObjectStorage getUploadMetadataForFilesParameters with | Ok returnValue -> () | Error error -> logToAnsiConsole Colors.Error $"Failed to upload changed files to object storage. {error}" diff --git a/src/Grace.CLI/Command/Maintenance.CLI.fs b/src/Grace.CLI/Command/Maintenance.CLI.fs index 111fccc..a8fc2d3 100644 --- a/src/Grace.CLI/Command/Maintenance.CLI.fs +++ b/src/Grace.CLI/Command/Maintenance.CLI.fs @@ -22,13 +22,71 @@ open System.Threading.Tasks open System.Collections.Generic open Grace.Shared.Parameters.Directory open Grace.Shared.Services +open Grace.Shared.Parameters.Storage module Maintenance = type CommonParameters() = inherit ParameterBase() + member val public OwnerId = String.Empty with get, set + member val public OwnerName: OwnerName = String.Empty with get, set + member val public OrganizationId = String.Empty with get, set + member val public OrganizationName: OrganizationName = String.Empty with get, set + member val public RepositoryId = String.Empty with get, set + member val public RepositoryName: RepositoryName = String.Empty with get, set module private Options = + let ownerId = + new Option( + "--ownerId", + IsRequired = false, + Description = "The repository's owner ID .", + Arity = ArgumentArity.ZeroOrOne, + getDefaultValue = (fun _ -> $"{Current().OwnerId}") + ) + + let ownerName = + new Option( + "--ownerName", + IsRequired = false, + Description = "The repository's owner name. [default: current owner]", + Arity = ArgumentArity.ExactlyOne + ) + + let organizationId = + new Option( + "--organizationId", + IsRequired = false, + Description = "The repository's organization ID .", + Arity = ArgumentArity.ZeroOrOne, + getDefaultValue = (fun _ -> $"{Current().OrganizationId}") + ) + + let organizationName = + new Option( + "--organizationName", + IsRequired = false, + Description = "The repository's organization name. [default: current organization]", + Arity = ArgumentArity.ZeroOrOne + ) + + let repositoryId = + new Option( + [| "--repositoryId"; "-r" |], + IsRequired = false, + Description = "The repository's ID .", + Arity = ArgumentArity.ExactlyOne, + getDefaultValue = (fun _ -> $"{Current().RepositoryId}") + ) + + let repositoryName = + new Option( + [| "--repositoryName"; "-n" |], + IsRequired = false, + Description = "The name of the repository. [default: current repository]", + Arity = ArgumentArity.ExactlyOne + ) + let listDirectories = new Option( "--listDirectories", @@ -165,13 +223,8 @@ module Maintenance = graceStatus.Index.Values, Constants.ParallelOptions, (fun localDirectoryVersion -> - if not <| objectCache.Index.ContainsKey(localDirectoryVersion.DirectoryVersionId) then - objectCache.Index.AddOrUpdate( - localDirectoryVersion.DirectoryVersionId, - (fun _ -> localDirectoryVersion), - (fun _ _ -> localDirectoryVersion) - ) - |> ignore + objectCache.Index.GetOrAdd(localDirectoryVersion.DirectoryVersionId, (fun _ -> localDirectoryVersion)) + |> ignore t4.Increment(incrementAmount)) ) @@ -205,12 +258,20 @@ module Maintenance = (fun fileVersions ct -> ValueTask( task { - let! graceResult = - Storage.FilesExistInObjectStorage - (fileVersions.Select(fun f -> f.Value.ToFileVersion).ToList()) - (getCorrelationId parseResult) + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (fileVersions |> Seq.map (fun kvp -> kvp.Value.ToFileVersion) |> Seq.toArray) + ) - match graceResult with + match! Storage.GetUploadMetadataForFiles getUploadMetadataForFilesParameters with | Ok graceReturnValue -> let uploadMetadata = graceReturnValue.ReturnValue // Increment the counter for the files that we don't have to upload. @@ -235,6 +296,7 @@ module Maintenance = let! result = Storage.SaveFileToObjectStorage + (Current().RepositoryId) fileVersion (upload.BlobUriWithSasToken) (getCorrelationId parseResult) @@ -392,12 +454,19 @@ module Maintenance = (fun fileVersions ct -> ValueTask( task { - let! graceResult = - Storage.FilesExistInObjectStorage - (fileVersions.Select(fun f -> f.Value.ToFileVersion).ToList()) - (getCorrelationId parseResult) - - match graceResult with + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = (fileVersions |> Seq.map (fun kvp -> kvp.Value.ToFileVersion) |> Seq.toArray) + ) + + match! Storage.GetUploadMetadataForFiles getUploadMetadataForFilesParameters with | Ok graceReturnValue -> let uploadMetadata = graceReturnValue.ReturnValue @@ -417,6 +486,7 @@ module Maintenance = let! result = Storage.SaveFileToObjectStorage + (Current().RepositoryId) fileVersion (upload.BlobUriWithSasToken) (getCorrelationId parseResult) @@ -588,12 +658,20 @@ module Maintenance = (fun fileVersions ct -> ValueTask( task { - let! graceResult = - Storage.FilesExistInObjectStorage - (fileVersions.Select(fun f -> f.Value.ToFileVersion).ToList()) - (getCorrelationId parseResult) + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (fileVersions |> Seq.map (fun kvp -> kvp.Value.ToFileVersion) |> Seq.toArray) + ) - match graceResult with + match! Storage.GetUploadMetadataForFiles getUploadMetadataForFilesParameters with | Ok graceReturnValue -> let uploadMetadata = graceReturnValue.ReturnValue // Increment the counter for the files that we don't have to upload. @@ -618,6 +696,7 @@ module Maintenance = let! result = Storage.SaveFileToObjectStorage + (Current().RepositoryId) fileVersion (upload.BlobUriWithSasToken) (getCorrelationId parseResult) @@ -775,12 +854,19 @@ module Maintenance = (fun fileVersions ct -> ValueTask( task { - let! graceResult = - Storage.FilesExistInObjectStorage - (fileVersions.Select(fun f -> f.Value.ToFileVersion).ToList()) - (getCorrelationId parseResult) - - match graceResult with + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = (fileVersions |> Seq.map (fun kvp -> kvp.Value.ToFileVersion) |> Seq.toArray) + ) + + match! Storage.GetUploadMetadataForFiles getUploadMetadataForFilesParameters with | Ok graceReturnValue -> let uploadMetadata = graceReturnValue.ReturnValue @@ -800,6 +886,7 @@ module Maintenance = let! result = Storage.SaveFileToObjectStorage + (Current().RepositoryId) fileVersion (upload.BlobUriWithSasToken) (getCorrelationId parseResult) @@ -990,35 +1077,50 @@ module Maintenance = :> Task) let Build = + let addCommonOptions (command: Command) = + command + |> addOption Options.ownerName + |> addOption Options.ownerId + |> addOption Options.organizationName + |> addOption Options.organizationId + |> addOption Options.repositoryId + |> addOption Options.repositoryName + let maintenanceCommand = new Command("maintenance", Description = "Performs various maintenance tasks.") maintenanceCommand.AddAlias("maint") let updateIndexCommand = new Command("update-index", Description = "Recreates the local Grace index file based on the current working directory contents.") + |> addCommonOptions updateIndexCommand.Handler <- UpdateIndex maintenanceCommand.AddCommand(updateIndexCommand) - let scanCommand = new Command("scan", Description = "Scans the working directory contents for changes.") + let scanCommand = + new Command("scan", Description = "Scans the working directory contents for changes.") + |> addCommonOptions scanCommand.Handler <- Scan maintenanceCommand.AddCommand(scanCommand) - let statsCommand = new Command("stats", Description = "Displays statistics about the current working directory.") + let statsCommand = + new Command("stats", Description = "Displays statistics about the current working directory.") + |> addCommonOptions statsCommand.Handler <- Stats maintenanceCommand.AddCommand(statsCommand) let listContentsCommand = new Command("list-contents", Description = "List directories and files from the Grace Status file.") + |> addCommonOptions |> addOption Options.listDirectories |> addOption Options.listFiles listContentsCommand.Handler <- ListContents maintenanceCommand.AddCommand(listContentsCommand) - let testCommand = new Command("test", Description = "Just a test.") + let testCommand = new Command("test", Description = "Just a test.") |> addCommonOptions testCommand.Handler <- Test maintenanceCommand.AddCommand(testCommand) diff --git a/src/Grace.CLI/Command/Reference.CLI.fs b/src/Grace.CLI/Command/Reference.CLI.fs index 4a992cf..20afdb6 100644 --- a/src/Grace.CLI/Command/Reference.CLI.fs +++ b/src/Grace.CLI/Command/Reference.CLI.fs @@ -35,6 +35,7 @@ open System.Security.Cryptography open System.Threading.Tasks open System.Text open System.Text.Json +open Grace.Shared.Parameters.Storage module Reference = open Grace.Shared.Validation.Common.Input @@ -637,6 +638,8 @@ module Reference = //let sha256Bytes = SHA256.HashData(Encoding.ASCII.GetBytes(rnd.NextInt64().ToString("x8"))) //let sha256Hash = Seq.fold (fun (sb: StringBuilder) currentByte -> // sb.Append(sprintf $"{currentByte:X2}")) (StringBuilder(sha256Bytes.Length)) sha256Bytes + let repositoryId = RepositoryId.Parse(parameters.RepositoryId) + if parseResult |> hasOutput then return! progress @@ -717,7 +720,22 @@ module Reference = let mutable lastFileUploadInstant = newGraceStatus.LastSuccessfulFileUpload if newFileVersions.Count() > 0 then - match! uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) with + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + match! uploadFilesToObjectStorage getUploadMetadataForFilesParameters with | Ok returnValue -> () //logToAnsiConsole Colors.Verbose $"Uploaded all files to object storage." | Error error -> logToAnsiConsole Colors.Error $"Error uploading files to object storage: {error.Error}" @@ -803,7 +821,22 @@ module Reference = .First(fun dv -> dv.Files.Exists(fun file -> file.RelativePath = relativePath)) .Files.First(fun file -> file.RelativePath = relativePath)) - let! uploadResult = uploadFilesToObjectStorage newFileVersions (getCorrelationId parseResult) + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (newFileVersions + |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) + |> Seq.toArray) + ) + + let! uploadResult = uploadFilesToObjectStorage getUploadMetadataForFilesParameters let saveParameters = SaveDirectoryVersionsParameters() saveParameters.DirectoryVersions <- newDirectoryVersions.Select(fun dv -> dv.ToDirectoryVersion).ToList() diff --git a/src/Grace.CLI/Command/Repository.CLI.fs b/src/Grace.CLI/Command/Repository.CLI.fs index 333fe47..a280b5b 100644 --- a/src/Grace.CLI/Command/Repository.CLI.fs +++ b/src/Grace.CLI/Command/Repository.CLI.fs @@ -26,12 +26,12 @@ open System.Threading open System.Threading.Tasks open Grace.Shared.Parameters.Directory open Spectre.Console.Json +open Grace.Shared.Parameters.Storage module Repository = type CommonParameters() = inherit ParameterBase() - member val public Name: RepositoryName = String.Empty with get, set member val public OwnerId = String.Empty with get, set member val public OwnerName: OwnerName = String.Empty with get, set member val public OrganizationId = String.Empty with get, set @@ -458,6 +458,8 @@ module Repository = match repositoryIsEmpty with | Ok isEmpty -> if isEmpty.ReturnValue = true then + let repositoryId = RepositoryId.Parse(parameters.RepositoryId) + if parseResult |> hasOutput then let! graceStatus = progress @@ -590,10 +592,23 @@ module Repository = (fun fileVersions ct -> ValueTask( task { + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = parameters.OwnerId, + OwnerName = parameters.OwnerName, + OrganizationId = parameters.OrganizationId, + OrganizationName = parameters.OrganizationName, + RepositoryId = parameters.RepositoryId, + RepositoryName = parameters.RepositoryName, + CorrelationId = getCorrelationId parseResult, + FileVersions = + (fileVersions + |> Seq.map (fun kvp -> kvp.Value.ToFileVersion) + |> Seq.toArray) + ) + let! graceResult = - Storage.FilesExistInObjectStorage - (fileVersions.Select(fun f -> f.Value.ToFileVersion).ToList()) - (getCorrelationId parseResult) + Storage.GetUploadMetadataForFiles getUploadMetadataForFilesParameters match graceResult with | Ok graceReturnValue -> @@ -624,6 +639,7 @@ module Repository = let! result = Storage.SaveFileToObjectStorage + repositoryId fileVersion (upload.BlobUriWithSasToken) (getCorrelationId parseResult) diff --git a/src/Grace.CLI/Command/Services.CLI.fs b/src/Grace.CLI/Command/Services.CLI.fs index 0861abc..17c366c 100644 --- a/src/Grace.CLI/Command/Services.CLI.fs +++ b/src/Grace.CLI/Command/Services.CLI.fs @@ -32,6 +32,9 @@ open System.Text.Json open System.Threading.Tasks open System.Reactive.Linq open MessagePack +open System.Threading +open Grace.Shared.Parameters +open Grace.Shared.Parameters.Storage module Services = @@ -204,7 +207,6 @@ module Services = return Some( LocalFileVersion.Create - (Current().RepositoryId) (RelativePath(normalizeFilePath relativePath)) (Sha256Hash shaValue) isBinary @@ -307,7 +309,13 @@ module Services = } /// Writes the Grace status file to disk. - let writeGraceStatusFile (graceStatus: GraceStatus) = task { do! serializeMessagePack (Current().GraceStatusFile) graceStatus } + let writeGraceStatusFile (graceStatus: GraceStatus) = + task { + Directory.CreateDirectory(Path.GetDirectoryName(Current().GraceStatusFile)) + |> ignore // No-op if the directory already exists + + do! serializeMessagePack (Current().GraceStatusFile) graceStatus + } /// Writes the Grace status file to disk. let writeGraceStatusFileOld (graceStatus: GraceStatus) = @@ -354,7 +362,13 @@ module Services = } /// Writes the Grace object cache file to disk. - let writeGraceObjectCacheFile (graceObjectCache: GraceObjectCache) = task { do! serializeMessagePack (Current().GraceObjectCacheFile) graceObjectCache } + let writeGraceObjectCacheFile (graceObjectCache: GraceObjectCache) = + task { + Directory.CreateDirectory(Path.GetDirectoryName(Current().GraceObjectCacheFile)) + |> ignore // No-op if the directory already exists + + do! serializeMessagePack (Current().GraceObjectCacheFile) graceObjectCache + } /// Writes the Grace object cache file to disk. let writeGraceObjectCacheFileOld (graceObjectCache: GraceObjectCache) = @@ -434,6 +448,8 @@ module Services = } //let processedThings = ConcurrentQueue() + let mutable newDirectoryVersionCount = 0 + let mutable existingDirectoryVersionCount = 0 /// Gathers all of the LocalDirectoryVersions and LocalFileVersions for the requested directory and its subdirectories, and returns them along with the Sha256Hash of the requested directory. let rec collectDirectoriesAndFiles @@ -520,6 +536,12 @@ module Services = newGraceStatus.Index.TryAdd(subdirectoryVersion.DirectoryVersionId, subdirectoryVersion) |> ignore + Interlocked.Increment(&newDirectoryVersionCount) |> ignore + + logToAnsiConsole + Colors.Important + $"****Created new DirectoryVersion: RelativePath: {normalizeFilePath subdirectoryRelativePath}; existing SHA256Hash: {existingSubdirectoryVersion.Sha256Hash}; new SHA256Hash: {sha256Hash}." + directories.Enqueue(subdirectoryVersion) else if parseResult |> isOutputFormat "Verbose" then @@ -530,6 +552,8 @@ module Services = newGraceStatus.Index.TryAdd(existingSubdirectoryVersion.DirectoryVersionId, existingSubdirectoryVersion) |> ignore + Interlocked.Increment(&existingDirectoryVersionCount) |> ignore + directories.Enqueue(existingSubdirectoryVersion) } )) @@ -549,6 +573,12 @@ module Services = $"In collectDirectoriesAndFiles: Processing {relativeDirectoryPath}: {files.Count} files, {directories.Count} directories." let sha256Hash = computeSha256ForDirectory relativeDirectoryPath directories files + + //if relativeDirectoryPath = Constants.RootDirectoryPath then + logToAnsiConsole + Colors.Verbose + $"In collectDirectoriesAndFiles: newDirectoryVersionCount: {newDirectoryVersionCount}; existingDirectoryVersionCount: {existingDirectoryVersionCount}." + return (directories, files, sha256Hash) } @@ -699,14 +729,15 @@ module Services = } /// Uploads all new or changed files from a directory to object storage. - let uploadFilesToObjectStorage (fileVersions: IEnumerable) (correlationId: string) = + let uploadFilesToObjectStorage (parameters: GetUploadMetadataForFilesParameters) = task { match Current().ObjectStorageProvider with - | ObjectStorageProvider.Unknown -> return Error(GraceError.Create (StorageError.getErrorMessage StorageError.NotImplemented) correlationId) + | ObjectStorageProvider.Unknown -> + return Error(GraceError.Create (StorageError.getErrorMessage StorageError.NotImplemented) parameters.CorrelationId) | AzureBlobStorage -> //logToAnsiConsole Colors.Verbose $"Uploading {fileVersions.Count()} files to object storage." - if fileVersions.Count() > 0 then - match! Storage.FilesExistInObjectStorage (fileVersions.Select(fun f -> f.ToFileVersion).ToList()) correlationId with + if parameters.FileVersions.Count() > 0 then + match! Storage.GetUploadMetadataForFiles parameters with | Ok graceReturnValue -> let filesToUpload = graceReturnValue.ReturnValue //logToAnsiConsole Colors.Verbose $"In Services.uploadFilesToObjectStorage(): filesToUpload: {serialize filesToUpload}." @@ -720,10 +751,15 @@ module Services = ValueTask( task { let fileVersion = - (fileVersions.First(fun f -> f.Sha256Hash = uploadMetadata.Sha256Hash)) - .ToFileVersion + (parameters.FileVersions.First(fun fileVersion -> fileVersion.Sha256Hash = uploadMetadata.Sha256Hash)) //logToAnsiConsole Colors.Verbose $"In Services.uploadFilesToObjectStorage(): Uploading {fileVersion.GetObjectFileName} to object storage." - match! Storage.SaveFileToObjectStorage fileVersion (uploadMetadata.BlobUriWithSasToken) correlationId with + match! + Storage.SaveFileToObjectStorage + (RepositoryId.Parse(parameters.RepositoryId)) + fileVersion + (uploadMetadata.BlobUriWithSasToken) + parameters.CorrelationId + with | Ok result -> () //logToAnsiConsole Colors.Verbose $"In Services.uploadFilesToObjectStorage(): Uploaded {fileVersion.GetObjectFileName} to object storage." | Error error -> errors.Enqueue(error) } @@ -731,34 +767,20 @@ module Services = ) if errors.Count = 0 then - return Ok(GraceReturnValue.Create true correlationId) + return Ok(GraceReturnValue.Create true parameters.CorrelationId) else // use Seq.fold to create a single error message from the ConcurrentQueue let errorMessage = errors |> Seq.fold (fun acc error -> $"{acc}\n{error.Error}") "" - let graceError = GraceError.Create (StorageError.getErrorMessage StorageError.FailedUploadingFilesToObjectStorage) correlationId + let graceError = + GraceError.Create (StorageError.getErrorMessage StorageError.FailedUploadingFilesToObjectStorage) parameters.CorrelationId return Error graceError |> enhance "Errors" errorMessage | Error error -> return Error error else - return Ok(GraceReturnValue.Create true correlationId) - | AWSS3 -> return Error(GraceError.Create (StorageError.getErrorMessage StorageError.NotImplemented) correlationId) - | GoogleCloudStorage -> return Error(GraceError.Create (StorageError.getErrorMessage StorageError.NotImplemented) correlationId) - } - - /// Uploads a new or changed file to object storage. - let uploadToServerAsync (fileVersion: FileVersion) correlationId = - task { - match! Storage.GetUploadUri fileVersion correlationId with - | Ok returnValue -> - match! Storage.SaveFileToObjectStorage fileVersion (Uri(returnValue.ReturnValue)) correlationId with - | Ok message -> - //printfn $"{message}" - return Ok message - | Error error -> - printfn $"{error}" - return Error error - | Error error -> return Error error + return Ok(GraceReturnValue.Create true parameters.CorrelationId) + | AWSS3 -> return Error(GraceError.Create (StorageError.getErrorMessage StorageError.NotImplemented) parameters.CorrelationId) + | GoogleCloudStorage -> return Error(GraceError.Create (StorageError.getErrorMessage StorageError.NotImplemented) parameters.CorrelationId) } /// Creates an updated LocalDirectoryVersion instance, with a new DirectoryId, based on changes to an existing one. @@ -813,7 +835,7 @@ module Services = | FileSystemEntryType.File -> false /// Determines if the given difference is for a file, instead of a directory. - let isFileChange difference = not (isDirectoryChange difference) + let isFileChange difference = not <| isDirectoryChange difference /// Gets a list of new or updated LocalDirectoryVersions that reflect changes in the working directory. /// @@ -1455,16 +1477,7 @@ module Services = //logToConsole $"Finished copyToObjectDirectory for {filePath}; isBinary: {isBinary}; moved temp file to object directory." let relativePath = Path.GetRelativePath(Current().RootDirectory, filePath) - return - Some( - FileVersion.Create - (Current().RepositoryId) - (RelativePath relativePath) - (Sha256Hash $"{sha256Hash}") - ("") - isBinary - (objectFilePathInfo.Length) - ) + return Some(FileVersion.Create (RelativePath relativePath) (Sha256Hash $"{sha256Hash}") ("") isBinary (objectFilePathInfo.Length)) else // If we do already have this exact version of the file, just delete the temp file. File.Delete(tempFilePath) diff --git a/src/Grace.CLI/Command/Watch.CLI.fs b/src/Grace.CLI/Command/Watch.CLI.fs index 9b1b81e..6e353fa 100644 --- a/src/Grace.CLI/Command/Watch.CLI.fs +++ b/src/Grace.CLI/Command/Watch.CLI.fs @@ -36,6 +36,7 @@ open System.Threading open System.Collections.Concurrent open Spectre.Console open System.Text +open Grace.Shared.Parameters.Storage module Watch = @@ -237,13 +238,15 @@ module Watch = } /// Copies a file from the working directory to the object directory, with its SHA-256 hash, and then uploads it to storage. - let copyFileToObjectDirectoryAndUploadToStorage fullPath correlationId = + let copyFileToObjectDirectoryAndUploadToStorage (getUploadMetadataForFilesParameters: GetUploadMetadataForFilesParameters) fullPath = task { //logToConsole $"*In fileChanged for {fullPath}." match! copyToObjectDirectory fullPath with | Some fileVersion -> - match! uploadToServerAsync fileVersion correlationId with - | Ok correlationId -> logToAnsiConsole Colors.Verbose $"File {fileVersion.GetObjectFileName} has been uploaded to storage." + getUploadMetadataForFilesParameters.FileVersions <- [| fileVersion |] + + match! uploadFilesToObjectStorage getUploadMetadataForFilesParameters with + | Ok returnValue -> logToAnsiConsole Colors.Verbose $"File {fileVersion.GetObjectFileName} has been uploaded to storage." | Error error -> logToAnsiConsole Colors.Error $"**Failed to upload {fileVersion.GetObjectFileName} to storage." | None -> () } @@ -298,11 +301,18 @@ module Watch = // Loop through no more than 50 files. Copy them to the objects directory, and upload them to storage. // In the incredibly rare event that more than 50 files have changed, we'll get 50-per-timer-tick, // and clear the queue quickly without overwhelming the system. + let getUploadMetadataForFilesParameters = + GetUploadMetadataForFilesParameters( + OwnerId = $"{Current().OwnerId}", + OrganizationId = $"{Current().OrganizationId}", + RepositoryId = $"{Current().RepositoryId}", + CorrelationId = correlationId + ) + for fileName in filesToProcess.Keys.Take(50) do if filesToProcess.TryRemove(fileName, &unitValue) then logToAnsiConsole Colors.Verbose $"Processing {fileName}. filesToProcess.Count: {filesToProcess.Count}." - - do! copyFileToObjectDirectoryAndUploadToStorage (FilePath fileName) correlationId + do! copyFileToObjectDirectoryAndUploadToStorage getUploadMetadataForFilesParameters (FilePath fileName) lastFileUploadInstant <- getCurrentInstant () graceStatus <- { graceStatus with LastSuccessfulFileUpload = lastFileUploadInstant } diff --git a/src/Grace.SDK/Storage.SDK.fs b/src/Grace.SDK/Storage.SDK.fs index b4bf788..4fbb81a 100644 --- a/src/Grace.SDK/Storage.SDK.fs +++ b/src/Grace.SDK/Storage.SDK.fs @@ -26,6 +26,7 @@ open System.Net open System.Net.Http.Json open System.Threading.Tasks open System.Text +open Grace.Shared.Parameters.Storage module Storage = @@ -98,15 +99,17 @@ module Storage = return Error(GraceError.Create (StorageError.getErrorMessage ObjectStorageException) correlationId) } - let FilesExistInObjectStorage (fileVersions: List) correlationId = + let GetUploadMetadataForFiles (parameters: GetUploadMetadataForFilesParameters) = task { + let correlationId = parameters.CorrelationId + try - if fileVersions.Count > 0 then + if parameters.FileVersions.Length > 0 then match Current().ObjectStorageProvider with | AzureBlobStorage -> let httpClient = getHttpClient correlationId - let serviceUrl = $"{Current().ServerUri}/storage/filesExistInObjectStorage" - let jsonContent = createJsonContent fileVersions + let serviceUrl = $"{Current().ServerUri}/storage/getUploadMetadataForFiles" + let jsonContent = createJsonContent parameters let! response = httpClient.PostAsync(serviceUrl, jsonContent) if response.IsSuccessStatusCode then @@ -120,7 +123,7 @@ module Storage = let fileVersionList = StringBuilder() - for fileVersion in fileVersions do + for fileVersion in parameters.FileVersions do fileVersionList.Append($"{fileVersion.RelativePath}; ") |> ignore return Error graceError |> enhance "fileVersions" $"{fileVersionList}" @@ -136,7 +139,13 @@ module Storage = let storageTransferOptions = StorageTransferOptions(MaximumConcurrency = Constants.ParallelOptions.MaxDegreeOfParallelism) - let SaveFileToObjectStorageWithMetadata (fileVersion: FileVersion) (blobUriWithSasToken: Uri) (metadata: Dictionary) correlationId = + let SaveFileToObjectStorageWithMetadata + (repositoryId: RepositoryId) + (fileVersion: FileVersion) + (blobUriWithSasToken: Uri) + (metadata: Dictionary) + correlationId + = task { try //logToConsole $"In SDK.Storage.SaveFileToObjectStorageWithMetadata: fileVersion.RelativePath: {fileVersion.RelativePath}." @@ -208,7 +217,7 @@ module Storage = returnValue.Properties.Add(nameof (Sha256Hash), $"{fileVersion.Sha256Hash}") returnValue.Properties.Add(nameof (RelativePath), $"{fileVersion.RelativePath}") - returnValue.Properties.Add(nameof (RepositoryId), $"{fileVersion.RepositoryId}") + returnValue.Properties.Add(nameof (RepositoryId), $"{repositoryId}") return Ok returnValue else let error = (GraceError.Create $"Failed to upload file {normalizedObjectFilePath} to object storage." correlationId) @@ -219,7 +228,7 @@ module Storage = returnValue.Properties.Add(nameof (Sha256Hash), $"{fileVersion.Sha256Hash}") returnValue.Properties.Add(nameof (RelativePath), $"{fileVersion.RelativePath}") - returnValue.Properties.Add(nameof (RepositoryId), $"{fileVersion.RepositoryId}") + returnValue.Properties.Add(nameof (RepositoryId), $"{repositoryId}") return Ok returnValue with ex -> let exceptionResponse = ExceptionResponse.Create ex @@ -232,47 +241,49 @@ module Storage = return Error(GraceError.Create (exceptionResponse.ToString()) correlationId) } - let SaveFileToObjectStorage (fileVersion: FileVersion) (blobUriWithSasToken: Uri) correlationId = - SaveFileToObjectStorageWithMetadata fileVersion blobUriWithSasToken (Dictionary()) correlationId + let SaveFileToObjectStorage (repositoryId: RepositoryId) (fileVersion: FileVersion) (blobUriWithSasToken: Uri) correlationId = + SaveFileToObjectStorageWithMetadata repositoryId fileVersion blobUriWithSasToken (Dictionary()) correlationId - let GetUploadUri (fileVersion: FileVersion) correlationId = + let GetUploadUri (parameters: GetUploadUriParameters) = task { try match Current().ObjectStorageProvider with - | ObjectStorageProvider.Unknown -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) correlationId) + | ObjectStorageProvider.Unknown -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) parameters.CorrelationId) | ObjectStorageProvider.AzureBlobStorage -> - let httpClient = getHttpClient correlationId + let httpClient = getHttpClient parameters.CorrelationId let serviceUrl = $"{Current().ServerUri}/storage/getUploadUri" - let jsonContent = createJsonContent fileVersion + let jsonContent = createJsonContent parameters let! response = httpClient.PostAsync(serviceUrl, jsonContent) let! blobUriWithSasToken = response.Content.ReadAsStringAsync() //logToConsole $"blobUriWithSasToken: {blobUriWithSasToken}" - return Ok(GraceReturnValue.Create blobUriWithSasToken correlationId) - | ObjectStorageProvider.AWSS3 -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) correlationId) - | ObjectStorageProvider.GoogleCloudStorage -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) correlationId) + return Ok(GraceReturnValue.Create blobUriWithSasToken parameters.CorrelationId) + | ObjectStorageProvider.AWSS3 -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) parameters.CorrelationId) + | ObjectStorageProvider.GoogleCloudStorage -> + return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) parameters.CorrelationId) with ex -> let exceptionResponse = ExceptionResponse.Create ex logToConsole $"exception: {exceptionResponse.ToString()}" - return Error(GraceError.Create (exceptionResponse.ToString()) correlationId) + return Error(GraceError.Create (exceptionResponse.ToString()) parameters.CorrelationId) } - let GetDownloadUri (fileVersion: FileVersion) correlationId = + let GetDownloadUri (parameters: GetDownloadUriParameters) = task { try match Current().ObjectStorageProvider with - | ObjectStorageProvider.Unknown -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) correlationId) + | ObjectStorageProvider.Unknown -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) parameters.CorrelationId) | ObjectStorageProvider.AzureBlobStorage -> - let httpClient = getHttpClient correlationId + let httpClient = getHttpClient parameters.CorrelationId let serviceUrl = $"{Current().ServerUri}/storage/getDownloadUri" - let jsonContent = createJsonContent fileVersion + let jsonContent = createJsonContent parameters let! response = httpClient.PostAsync(serviceUrl, jsonContent) let! blobUriWithSasToken = response.Content.ReadAsStringAsync() //logToConsole $"blobUriWithSasToken: {blobUriWithSasToken}" - return Ok(GraceReturnValue.Create blobUriWithSasToken correlationId) - | ObjectStorageProvider.AWSS3 -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) correlationId) - | ObjectStorageProvider.GoogleCloudStorage -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) correlationId) + return Ok(GraceReturnValue.Create blobUriWithSasToken parameters.CorrelationId) + | ObjectStorageProvider.AWSS3 -> return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) parameters.CorrelationId) + | ObjectStorageProvider.GoogleCloudStorage -> + return Error(GraceError.Create (StorageError.getErrorMessage NotImplemented) parameters.CorrelationId) with ex -> let exceptionResponse = ExceptionResponse.Create ex logToConsole $"exception: {exceptionResponse.ToString()}" - return Error(GraceError.Create (exceptionResponse.ToString()) correlationId) + return Error(GraceError.Create (exceptionResponse.ToString()) parameters.CorrelationId) } diff --git a/src/Grace.Server/DirectoryVersion.Server.fs b/src/Grace.Server/DirectoryVersion.Server.fs index 5589bc3..bdfa779 100644 --- a/src/Grace.Server/DirectoryVersion.Server.fs +++ b/src/Grace.Server/DirectoryVersion.Server.fs @@ -275,16 +275,24 @@ module DirectoryVersion = (fun directoryVersion ct -> ValueTask( task { - // Check if the directory version exists. If it doesn't, create it. - let directoryVersionActor = DirectoryVersion.CreateActorProxy directoryVersion.DirectoryVersionId correlationId + try + // Check if the directory version exists. If it doesn't, create it. + let directoryVersionActor = DirectoryVersion.CreateActorProxy directoryVersion.DirectoryVersionId correlationId - let! exists = directoryVersionActor.Exists parameters.CorrelationId - //logToConsole $"In SaveDirectoryVersions: {dv.DirectoryId} exists: {exists}" - if not <| exists then - let! createResult = - directoryVersionActor.Handle (DirectoryVersion.Create directoryVersion) (createMetadata context) + let! exists = directoryVersionActor.Exists parameters.CorrelationId + //logToConsole $"In SaveDirectoryVersions: {dv.DirectoryId} exists: {exists}" + if not <| exists then + let! createResult = + directoryVersionActor.Handle (DirectoryVersion.Create directoryVersion) (createMetadata context) - results.Enqueue(createResult) + results.Enqueue(createResult) + with ex -> + let exceptionResponse = Utilities.ExceptionResponse.Create ex + + logToConsole + $"****Error in SaveDirectoryVersions: directoryVersion.Directories.Count: {directoryVersion.Directories.Count}; directoryVersion.Files.Count: {directoryVersion.Files.Count}." + + logToConsole $"{exceptionResponse}" } )) ) diff --git a/src/Grace.Server/Middleware/ValidateIds.Middleware.fs b/src/Grace.Server/Middleware/ValidateIds.Middleware.fs index 8ea0e0a..b3dfac9 100644 --- a/src/Grace.Server/Middleware/ValidateIds.Middleware.fs +++ b/src/Grace.Server/Middleware/ValidateIds.Middleware.fs @@ -110,8 +110,9 @@ type ValidateIdsMiddleware(next: RequestDelegate) = // ----------------------------------------------------------------------------------------------------- // On the way in... -#if DEBUG let startTime = getCurrentInstant () + +#if DEBUG let middlewareTraceHeader = context.Request.Headers["X-MiddlewareTraceIn"] context.Request.Headers["X-MiddlewareTraceIn"] <- $"{middlewareTraceHeader}{nameof (ValidateIdsMiddleware)} --> " diff --git a/src/Grace.Server/ReminderService.Server.fs b/src/Grace.Server/ReminderService.Server.fs index defaea1..c1cbae7 100644 --- a/src/Grace.Server/ReminderService.Server.fs +++ b/src/Grace.Server/ReminderService.Server.fs @@ -33,15 +33,23 @@ module ReminderService = type ReminderService() = inherit BackgroundService() - let mutable reminderCount = 0 - let defaultReminderCount = 1000 - let timer = TimeSpan.FromSeconds(10.0) + let defaultReminderBatchSize = 5000 + let timer = TimeSpan.FromSeconds(60.0) let log = loggerFactory.CreateLogger("ReminderService.Server") + let reminderBatchSize = + let mutable reminderBatchSize = 0 + let enviromentValue = Configuration().Item(EnvironmentVariables.GraceReminderBatchSize) + + if Int32.TryParse(enviromentValue, &reminderBatchSize) then + reminderBatchSize + else + defaultReminderBatchSize + /// Retrieves reminders from storage. let retrieveReminders (cancellationToken: CancellationToken) = task { - let reminders = List(reminderCount) + let reminders = List(reminderBatchSize) match actorStateStorageProvider with | Unknown -> () @@ -54,7 +62,7 @@ module ReminderService = let queryDefinition = QueryDefinition(queryText) - .WithParameter("@maxCount", reminderCount) + .WithParameter("@maxCount", reminderBatchSize) .WithParameter("@class", nameof (ReminderDto)) use iterator = ApplicationContext.cosmosContainer.GetItemQueryIterator(queryDefinition) @@ -171,25 +179,12 @@ module ReminderService = override this.StartAsync(cancellationToken: CancellationToken) = log.LogInformation("{CurrentInstant}: Node: {HostName}; ReminderService is starting.", getCurrentInstantExtended (), getMachineName) - let enviromentValue = Configuration().Item(EnvironmentVariables.GraceReminderCount) - - if Int32.TryParse(enviromentValue, &reminderCount) then - log.LogInformation( - "{CurrentInstant}: Node: {HostName}; Reminder count set to {reminderCount}.", - getCurrentInstantExtended (), - getMachineName, - reminderCount - ) - else - reminderCount <- defaultReminderCount - - log.LogInformation( - "{CurrentInstant}: Node: {HostName}; Reminder count set to default value of {reminderCount}. You can set the `{GraceReminderCount}` environment variable to change the value.", - getCurrentInstantExtended (), - getMachineName, - reminderCount, - EnvironmentVariables.GraceReminderCount - ) + log.LogInformation( + "{CurrentInstant}: Node: {HostName}; Reminder batch size set to {reminderBatchSize}.", + getCurrentInstantExtended (), + getMachineName, + reminderBatchSize + ) base.StartAsync(cancellationToken) diff --git a/src/Grace.Server/Startup.Server.fs b/src/Grace.Server/Startup.Server.fs index bd771c1..8b610fb 100644 --- a/src/Grace.Server/Startup.Server.fs +++ b/src/Grace.Server/Startup.Server.fs @@ -249,10 +249,21 @@ module Application = subRoute "/storage" [ POST - [ route "/filesExistInObjectStorage" Storage.FilesExistInObjectStorage + [ route "/getUploadMetadataForFiles" Storage.GetUploadMetadataForFiles + |> addMetadata typeof route "/getDownloadUri" Storage.GetDownloadUri - route "/getUploadUri" Storage.GetUploadUri - route "/deleteAllFromCosmosDb" Storage.DeleteAllFromCosmosDB ] ] ] + |> addMetadata typeof + route "/getUploadUri" Storage.GetUploadUris + |> addMetadata typeof ] ] + subRoute + "/admin" + [ POST + [ +#if DEBUG + route "/deleteAllFromCosmosDB" Storage.DeleteAllFromCosmosDB + route "/deleteAllRemindersFromCosmosDB" Storage.DeleteAllRemindersFromCosmosDB +#endif + ] ] ] let notFoundHandler = "Not Found" |> text |> RequestErrors.notFound diff --git a/src/Grace.Server/Storage.Server.fs b/src/Grace.Server/Storage.Server.fs index ae6d031..39e6b61 100644 --- a/src/Grace.Server/Storage.Server.fs +++ b/src/Grace.Server/Storage.Server.fs @@ -13,6 +13,7 @@ open Grace.Actors.Services open Grace.Server.ApplicationContext open Grace.Server.Services open Grace.Shared.Dto.Repository +open Grace.Shared.Parameters.Storage open Grace.Shared.Types open Grace.Shared.Utilities open Grace.Shared @@ -29,6 +30,8 @@ open System.IO open System.Text open Azure.Storage open System.Diagnostics +open System.Reflection.Metadata +open System.Net.Http.Json module Storage = @@ -85,43 +88,58 @@ module Storage = let correlationId = (getCorrelationId context) try - let! fileVersion = context.BindJsonAsync() - let repositoryActor = Repository.CreateActorProxy fileVersion.RepositoryId correlationId + let! parameters = context.BindJsonAsync() + let repositoryActor = Repository.CreateActorProxy (RepositoryId.Parse(parameters.RepositoryId)) correlationId let! repositoryDto = repositoryActor.Get correlationId - match! getReadSharedAccessSignature repositoryDto fileVersion correlationId with + match! getReadSharedAccessSignature repositoryDto parameters.FileVersion correlationId with | Ok downloadUri -> context.SetStatusCode StatusCodes.Status200OK //context.GetLogger().LogTrace("fileVersion: {fileVersion.RelativePath}; downloadUri: {downloadUri}", [| fileVersion.RelativePath, downloadUri |]) - logToConsole $"fileVersion: {fileVersion.RelativePath}; downloadUri: {downloadUri}" + logToConsole $"fileVersion: {parameters.FileVersion.RelativePath}; downloadUri: {downloadUri}" return! context.WriteStringAsync $"{downloadUri}" | Error error -> context.SetStatusCode StatusCodes.Status500InternalServerError - logToConsole $"Error generating download Uri: fileVersion: {fileVersion.RelativePath}; Error: {error}" + logToConsole $"Error generating download Uri: fileVersion: {parameters.FileVersion.RelativePath}; Error: {error}" - return! context.WriteStringAsync $"Error creating download uri for {fileVersion.GetObjectFileName}." + return! context.WriteStringAsync $"Error creating download uri for {parameters.FileVersion.GetObjectFileName}." with ex -> context.SetStatusCode StatusCodes.Status500InternalServerError return! context.WriteTextAsync $"Error in {context.Request.Path} at {DateTime.Now.ToLongTimeString()}." } /// Gets an upload URI for the specified file version that can be used by a Grace client. - let GetUploadUri: HttpHandler = + let GetUploadUris: HttpHandler = fun (next: HttpFunc) (context: HttpContext) -> task { let correlationId = getCorrelationId context + let graceIds = getGraceIds context + let uris = Dictionary() try - let! fileVersion = context.BindJsonAsync() - let repositoryActor = Repository.CreateActorProxy fileVersion.RepositoryId correlationId + let! parameters = context.BindJsonAsync() + let repositoryActor = Repository.CreateActorProxy (RepositoryId.Parse(parameters.RepositoryId)) correlationId let! repositoryDto = repositoryActor.Get correlationId - let! uploadUri = getWriteSharedAccessSignature repositoryDto fileVersion correlationId - context.SetStatusCode StatusCodes.Status200OK - log.LogDebug("In GetUploadUri(): fileVersion.RelativePath: {relativePath}; uploadUri: {uploadUri}", fileVersion.RelativePath, uploadUri) + for fileVersion in parameters.FileVersions do + let! uploadUri = getWriteSharedAccessSignature repositoryDto fileVersion correlationId + uris.Add(fileVersion.RelativePath, uploadUri) + + if log.IsEnabled(LogLevel.Debug) then + let sb = stringBuilderPool.Get() + + try + for kvp in uris do + sb.AppendLine($"fileVersion: {kvp.Key}; uploadUri: {kvp.Value}") |> ignore - return! context.WriteStringAsync $"{uploadUri}" + log.LogDebug("In GetUploadUri(): Created {count} uri's for these files: {uploadUris}", uris.Count, sb.ToString()) + finally + stringBuilderPool.Return(sb) + + context.SetStatusCode StatusCodes.Status200OK + let jsonContent = JsonContent.Create(uris) + return! context.WriteJsonAsync jsonContent with ex -> context.SetStatusCode StatusCodes.Status500InternalServerError logToConsole $"Exception in GetUploadUri: {(ExceptionResponse.Create ex)}" @@ -129,25 +147,28 @@ module Storage = return! context.WriteTextAsync $"{getCurrentInstantExtended ()} Error in {context.Request.Path} at {DateTime.Now.ToLongTimeString()}." } - /// Checks if a list of files already exists in object storage, and if any do not, return a URL that the client can use to upload the file. - let FilesExistInObjectStorage: HttpHandler = + /// Checks if a list of files already exists in object storage, and if any do not, return Uri's that the client can use to upload the file. + let GetUploadMetadataForFiles: HttpHandler = fun (next: HttpFunc) (context: HttpContext) -> task { let correlationId = getCorrelationId context + let graceIds = getGraceIds context try - let! fileVersions = context.BindJsonAsync() - Activity.Current.SetTag("fileVersions.Count", $"{fileVersions.Count}") |> ignore + let! parameters = context.BindJsonAsync() - if fileVersions.Length > 0 then - let repositoryActor = Repository.CreateActorProxy fileVersions[0].RepositoryId correlationId + Activity.Current.SetTag("fileVersions.Count", $"{parameters.FileVersions.Length}") + |> ignore + + if parameters.FileVersions.Length > 0 then + let repositoryActor = Repository.CreateActorProxy (RepositoryId.Parse(graceIds.RepositoryId)) correlationId let! repositoryDto = repositoryActor.Get correlationId let uploadMetadata = ConcurrentQueue() do! Parallel.ForEachAsync( - fileVersions, + parameters.FileVersions, Constants.ParallelOptions, (fun fileVersion ct -> ValueTask( @@ -168,7 +189,7 @@ module Storage = context .GetLogger() .LogInformation( - $"{getCurrentInstantExtended ()} Received {fileVersions.Count} FileVersions; Returning {uploadMetadata.Count} uploadMetadata records." + $"{getCurrentInstantExtended ()} Received {parameters.FileVersions.Count} FileVersions; Returning {uploadMetadata.Count} uploadMetadata records." ) return! @@ -191,33 +212,92 @@ module Storage = fun (next: HttpFunc) (context: HttpContext) -> task { #if DEBUG - context - .GetLogger() - .LogWarning("{CurrentInstant} Deleting all rows from Cosmos DB.", getCurrentInstantExtended ()) + let correlationId = getCorrelationId context + let log = context.GetLogger() + + log.LogWarning("{CurrentInstant} Deleting all rows from Cosmos DB.", getCurrentInstantExtended ()) - let! failed = Services.deleteAllFromCosmosDB () + let! failed = deleteAllFromCosmosDb () if failed.Count = 0 then - context - .GetLogger() - .LogWarning("{CurrentInstant} Deleted all rows from Cosmos DB.", getCurrentInstantExtended ()) + log.LogWarning("{CurrentInstant} Succeeded deleting all rows from CosmosDB.", getCurrentInstantExtended ()) return! context - |> result200Ok (GraceReturnValue.Create "Deleted all rows from Cosmos DB." (getCorrelationId context)) + |> result200Ok (GraceReturnValue.Create "Succeeded deleting all rows from Cosmos DB." (getCorrelationId context)) else - let sb = StringBuilder() + let sb = stringBuilderPool.Get() + + try + for fail in failed do + sb.AppendLine(fail) |> ignore + + log.LogWarning( + "{CurrentInstant} Failed to delete all rows from Cosmos DB. Failures: {failedCount}.", + getCurrentInstantExtended (), + failed.Count + ) + + log.LogWarning(sb.ToString()) + + return! + context + |> result500ServerError ( + GraceError.Create + $"Failed to delete all rows from Cosmos DB. Failures: {failed.Count}.{Environment.NewLine}{sb.ToString()}" + (getCorrelationId context) + ) + finally + stringBuilderPool.Return(sb) +#else + return! context |> result404NotFound +#endif + } + + /// Deletes all reminders from Cosmos DB. After calling, the web connection will time-out, but the method will continue to run until Cosmos DB is empty. + /// + /// **** This method is implemented only in Debug configuration. It is a no-op in Release configuration. **** + let DeleteAllRemindersFromCosmosDB: HttpHandler = + fun (next: HttpFunc) (context: HttpContext) -> + task { +#if DEBUG + let correlationId = getCorrelationId context + let log = context.GetLogger() - for fail in failed do - sb.AppendLine(fail) |> ignore + log.LogWarning("{CurrentInstant} Deleting all reminders from Cosmos DB.", getCurrentInstantExtended ()) + + let! failed = deleteAllRemindersFromCosmosDb () + + if failed.Count = 0 then + log.LogWarning("{CurrentInstant} Succeeded deleting all reminders from CosmosDB.", getCurrentInstantExtended ()) return! context - |> result500ServerError ( - GraceError.Create - $"Failed to delete all rows from Cosmos DB. Failures: {failed.Count}.{Environment.NewLine}{sb.ToString()}" - (getCorrelationId context) + |> result200Ok (GraceReturnValue.Create "Succeeded deleting all reminders from Cosmos DB." (getCorrelationId context)) + else + let sb = stringBuilderPool.Get() + + try + for fail in failed do + sb.AppendLine(fail) |> ignore + + log.LogWarning( + "{CurrentInstant} Failed to delete all reminders from Cosmos DB. Failures: {failedCount}.", + getCurrentInstantExtended (), + failed.Count ) + + log.LogWarning(sb.ToString()) + + return! + context + |> result500ServerError ( + GraceError.Create + $"Failed to delete all reminders from Cosmos DB. Failures: {failed.Count}.{Environment.NewLine}{sb.ToString()}" + (getCorrelationId context) + ) + finally + stringBuilderPool.Return(sb) #else return! context |> result404NotFound #endif diff --git a/src/Grace.Shared/Constants.Shared.fs b/src/Grace.Shared/Constants.Shared.fs index 26c7640..028ac25 100644 --- a/src/Grace.Shared/Constants.Shared.fs +++ b/src/Grace.Shared/Constants.Shared.fs @@ -194,7 +194,7 @@ module Constants = /// The environment variable that contains the maximum number of reminders that each Grace instance should retrieve from the database and publish for processing. [] - let GraceReminderCount = "graceremindercount" + let GraceReminderBatchSize = "gracereminderbatchsize" /// The default CacheControl header for object storage. [] diff --git a/src/Grace.Shared/Grace.Shared.fsproj b/src/Grace.Shared/Grace.Shared.fsproj index d11e380..6552a92 100644 --- a/src/Grace.Shared/Grace.Shared.fsproj +++ b/src/Grace.Shared/Grace.Shared.fsproj @@ -35,6 +35,7 @@ + diff --git a/src/Grace.Shared/Parameters/Storage.Parameters.fs b/src/Grace.Shared/Parameters/Storage.Parameters.fs new file mode 100644 index 0000000..4b7da81 --- /dev/null +++ b/src/Grace.Shared/Parameters/Storage.Parameters.fs @@ -0,0 +1,34 @@ +namespace Grace.Shared.Parameters + +open Grace.Shared.Parameters.Common +open Grace.Shared.Types +open System + +module Storage = + + /// Parameters used by multiple endpoints in the /diff path. + type StorageParameters() = + inherit CommonParameters() + member val public OwnerId = String.Empty with get, set + member val public OwnerName: OwnerName = String.Empty with get, set + member val public OrganizationId = String.Empty with get, set + member val public OrganizationName: OrganizationName = String.Empty with get, set + member val public RepositoryId = String.Empty with get, set + member val public RepositoryName: RepositoryName = String.Empty with get, set + + /// Parameters used by the /diff/populate endpoint. + type DeleteAllParameters() = + inherit StorageParameters() + + /// Parameters used by the /diff/get endpoint. + type GetUploadUriParameters() = + inherit StorageParameters() + member val public FileVersions = Array.empty with get, set + + type GetDownloadUriParameters() = + inherit StorageParameters() + member val public FileVersion = FileVersion.Default with get, set + + type GetUploadMetadataForFilesParameters() = + inherit StorageParameters() + member val public FileVersions = Array.empty with get, set diff --git a/src/Grace.Shared/Types.Shared.fs b/src/Grace.Shared/Types.Shared.fs index 7bfeb52..2a8a348 100644 --- a/src/Grace.Shared/Types.Shared.fs +++ b/src/Grace.Shared/Types.Shared.fs @@ -137,7 +137,7 @@ module Types = [] type FileVersion = { Class: string - RepositoryId: RepositoryId + //RepositoryId: RepositoryId RelativePath: RelativePath Sha256Hash: Sha256Hash IsBinary: bool @@ -146,7 +146,7 @@ module Types = BlobUri: string } static member Create - (repositoryId: RepositoryId) + //(repositoryId: RepositoryId) (relativePath: RelativePath) (sha256Hash: Sha256Hash) (blobUri: string) @@ -154,7 +154,7 @@ module Types = (size: int64) = { Class = "FileVersion" - RepositoryId = repositoryId + //RepositoryId = repositoryId RelativePath = RelativePath(normalizeFilePath $"{relativePath}") Sha256Hash = sha256Hash BlobUri = blobUri @@ -162,9 +162,11 @@ module Types = Size = size CreatedAt = getCurrentInstant () } + static member Default = FileVersion.Create String.Empty String.Empty String.Empty false 0L + /// Converts a FileVersion to a LocalFileVersion. member this.ToLocalFileVersion lastWriteTimeUtc = - LocalFileVersion.Create this.RepositoryId this.RelativePath this.Sha256Hash this.IsBinary this.Size this.CreatedAt true lastWriteTimeUtc + LocalFileVersion.Create this.RelativePath this.Sha256Hash this.IsBinary this.Size this.CreatedAt true lastWriteTimeUtc /// Get the object directory file name, which includes the SHA256 Hash value. Example: hello.js -> hello_04bef0a4b298de9c02930234.js member this.GetObjectFileName = getObjectFileName this.RelativePath this.Sha256Hash @@ -177,8 +179,8 @@ module Types = and [] LocalFileVersion = { [] Class: string - [] - RepositoryId: RepositoryId + //[] + //RepositoryId: RepositoryId [] RelativePath: RelativePath [] @@ -197,7 +199,7 @@ module Types = static member GetKnownTypes() = GetKnownTypes() static member Create - (repositoryId: RepositoryId) + //(repositoryId: RepositoryId) (relativePath: RelativePath) (sha256Hash: Sha256Hash) (isBinary: bool) @@ -207,7 +209,7 @@ module Types = (lastWriteTimeUtc: DateTime) = { Class = "LocalFileVersion" - RepositoryId = repositoryId + //RepositoryId = repositoryId RelativePath = RelativePath(normalizeFilePath $"{relativePath}") Sha256Hash = sha256Hash IsBinary = isBinary @@ -218,7 +220,7 @@ module Types = /// Converts a LocalFileVersion to a FileVersion. NOTE: at this point, we don't know the BlobUri. [] - member this.ToFileVersion = FileVersion.Create this.RepositoryId this.RelativePath this.Sha256Hash String.Empty this.IsBinary this.Size + member this.ToFileVersion = FileVersion.Create this.RelativePath this.Sha256Hash String.Empty this.IsBinary this.Size /// Get the object directory file name, which includes the SHA256 Hash value. Example: hello.js -> hello_04bef0a4b298de9c02930234.js [] diff --git a/src/Grace.sln b/src/Grace.sln index fbab279..1d3a8f5 100644 --- a/src/Grace.sln +++ b/src/Grace.sln @@ -68,9 +68,7 @@ Global {72660935-7140-4D48-9BF2-020A790AC747}.Release|Any CPU.ActiveCfg = Release|Any CPU {72660935-7140-4D48-9BF2-020A790AC747}.Release|Any CPU.Build.0 = Release|Any CPU {17B590BA-543E-49B1-9EB7-262CE73D86FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {17B590BA-543E-49B1-9EB7-262CE73D86FE}.Debug|Any CPU.Build.0 = Debug|Any CPU {17B590BA-543E-49B1-9EB7-262CE73D86FE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {17B590BA-543E-49B1-9EB7-262CE73D86FE}.Release|Any CPU.Build.0 = Release|Any CPU {B31CE180-594F-46CF-95FE-4A482EFEC24D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B31CE180-594F-46CF-95FE-4A482EFEC24D}.Debug|Any CPU.Build.0 = Debug|Any CPU {B31CE180-594F-46CF-95FE-4A482EFEC24D}.Release|Any CPU.ActiveCfg = Release|Any CPU