From 087a1e5fea7db239f8d0fc66b82015924510df31 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:57:25 +0000 Subject: [PATCH 01/58] moves checkout step to input section of template context --- .../include-publish-npm-package-deployment.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 7b276394ef02..83b4214be9a3 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -46,6 +46,13 @@ jobs: displayName: Publish ${{ parameters.environment }} pool: ${{ parameters.pool }} environment: ${{ parameters.environment }} + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: checkout + repository: self + persistCredentials: true # Necessary for creation of git tags to work workspace: clean: all variables: @@ -55,9 +62,9 @@ jobs: runOnce: deploy: steps: - - checkout: self - clean: true - persistCredentials: true # Necessary for creation of git tags to work + # - checkout: self + # clean: true + # persistCredentials: true # Necessary for creation of git tags to work - download: current artifact: pack - template: /tools/pipelines/templates/include-use-node-version.yml@self From e8a96897745026eb00e344fa7b20803430d1493c Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:58:52 +0000 Subject: [PATCH 02/58] remove commented section --- .../templates/include-publish-npm-package-deployment.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 83b4214be9a3..23e64b907e70 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -62,9 +62,6 @@ jobs: runOnce: deploy: steps: - # - checkout: self - # clean: true - # persistCredentials: true # Necessary for creation of git tags to work - download: current artifact: pack - template: /tools/pipelines/templates/include-use-node-version.yml@self From b123c161cc87f1844c4e98c73b524c5b5730f412 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 21 Feb 2025 17:14:55 +0000 Subject: [PATCH 03/58] Initial upgrade to 1es release deployment job & moves checkout step to separate build step --- ...include-publish-npm-package-deployment.yml | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 23e64b907e70..97ac51ef57e7 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -42,7 +42,28 @@ parameters: default: $(Pipeline.Workspace)/.pnpm-store jobs: + +- job: buildJob1 + steps: + # Checkout the repository + - checkout: self + persistCredentials: true # Necessary for creation of git tags to work + # Copy required files to the artifact staging directory + - task: CopyFiles@2 + inputs: + SourceFolder: $(System.DefaultWorkingDirectory) + Contents: '**/*.json' + TargetFolder: $(Build.ArtifactStagingDirectory) + # Publish the artifact containing the required files from the repository + # Alternatively, you can use an `output` + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: MyDrop + - deployment: publish_${{ replace(parameters.environment, '-', '_') }} + # Depend on the build job + dependsOn: buildJob1 displayName: Publish ${{ parameters.environment }} pool: ${{ parameters.pool }} environment: ${{ parameters.environment }} @@ -50,9 +71,8 @@ jobs: type: releaseJob isProduction: true inputs: - - input: checkout - repository: self - persistCredentials: true # Necessary for creation of git tags to work + - input: pipelineArtifact + artifactName: MyDrop workspace: clean: all variables: From f78198f8e2388cfc40f6be587017d430f02058a8 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 21 Feb 2025 17:27:20 +0000 Subject: [PATCH 04/58] Moves all artifact downloads to templateContext --- .../include-publish-npm-package-deployment.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 97ac51ef57e7..30f57481d193 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -73,6 +73,9 @@ jobs: inputs: - input: pipelineArtifact artifactName: MyDrop + - input: pipelineArtifact + artifactName: pack + buildType: current workspace: clean: all variables: @@ -82,8 +85,13 @@ jobs: runOnce: deploy: steps: - - download: current - artifact: pack + # - task: DownloadPipelineArtifact@2 + # inputs: + # buildType: 'current' + # targetPath: '$(Pipeline.Workspace)' + # artifactName: pack + # - download: current + # artifact: pack - template: /tools/pipelines/templates/include-use-node-version.yml@self - template: /tools/pipelines/templates/include-install-build-tools.yml@self parameters: From 66dc38954fe012bbb27ef29e57895fe97e1d27c8 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 21 Feb 2025 18:30:43 +0000 Subject: [PATCH 05/58] adjustments to pipeline artifact download locations --- .../templates/include-publish-npm-package-deployment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 30f57481d193..d42888f7d31a 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -73,9 +73,12 @@ jobs: inputs: - input: pipelineArtifact artifactName: MyDrop + destinationPath: ${{ parameters.buildDirectory }} - input: pipelineArtifact artifactName: pack buildType: current + destinationPath: $(Pipeline.Workspace)/pack + workspace: clean: all variables: From 0b952973d4bab0688e1cff2387a50fe2fe321abb Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 21 Feb 2025 18:32:28 +0000 Subject: [PATCH 06/58] removes whitesapce --- .../templates/include-publish-npm-package-deployment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index d42888f7d31a..0c040a3c9efb 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -78,7 +78,6 @@ jobs: artifactName: pack buildType: current destinationPath: $(Pipeline.Workspace)/pack - workspace: clean: all variables: From 86d94e500ab117f23d213447c3df62c227777440 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 21 Feb 2025 19:30:16 +0000 Subject: [PATCH 07/58] switching destinationPath to targetPath syntax for inputs --- .../templates/include-publish-npm-package-deployment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 0c040a3c9efb..3e13773d28f9 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -73,11 +73,11 @@ jobs: inputs: - input: pipelineArtifact artifactName: MyDrop - destinationPath: ${{ parameters.buildDirectory }} + targetPath: ${{ parameters.buildDirectory }} - input: pipelineArtifact artifactName: pack buildType: current - destinationPath: $(Pipeline.Workspace)/pack + targetPath: $(Pipeline.Workspace)/pack workspace: clean: all variables: From 154db473b601175596af048a295c8897eb954cc2 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 21 Feb 2025 20:38:48 +0000 Subject: [PATCH 08/58] Updates copy step file selection and changes prerequisite job name --- .../templates/include-publish-npm-package-deployment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 3e13773d28f9..50494c3f035f 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -43,7 +43,7 @@ parameters: jobs: -- job: buildJob1 +- job: checkoutAndUploadThisRepoAsArtifact steps: # Checkout the repository - checkout: self @@ -52,7 +52,7 @@ jobs: - task: CopyFiles@2 inputs: SourceFolder: $(System.DefaultWorkingDirectory) - Contents: '**/*.json' + Contents: '**/*' TargetFolder: $(Build.ArtifactStagingDirectory) # Publish the artifact containing the required files from the repository # Alternatively, you can use an `output` @@ -63,7 +63,7 @@ jobs: - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job - dependsOn: buildJob1 + dependsOn: checkoutAndUploadThisRepoAsArtifact displayName: Publish ${{ parameters.environment }} pool: ${{ parameters.pool }} environment: ${{ parameters.environment }} From eef259eb1a017e7b7a4262a28c88577ef304f9a4 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 24 Feb 2025 17:34:28 +0000 Subject: [PATCH 09/58] adds comments and small cleanup --- .../include-publish-npm-package-deployment.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 50494c3f035f..832f4bca0ef0 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -43,6 +43,9 @@ parameters: jobs: + # This job is necessary to be compliant with 1ES. + # 1ES does not allow checking out a code repo within a deployment job so instead we use this pattern + # of checking out the repo in separate job and uploading it as a pipeline artifact. - job: checkoutAndUploadThisRepoAsArtifact steps: # Checkout the repository @@ -55,11 +58,10 @@ jobs: Contents: '**/*' TargetFolder: $(Build.ArtifactStagingDirectory) # Publish the artifact containing the required files from the repository - # Alternatively, you can use an `output` - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: $(Build.ArtifactStagingDirectory) - artifactName: MyDrop + artifactName: fluidFrameworkRepo - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job @@ -67,12 +69,15 @@ jobs: displayName: Publish ${{ parameters.environment }} pool: ${{ parameters.pool }} environment: ${{ parameters.environment }} + # This templateContext section is necessary for 1ES compliance for deployment jobs. + # see https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/releasepipelines/releaseworkflows/releasejob?tabs=combined-pipeline for more info. templateContext: type: releaseJob isProduction: true + # This section downloads initial required artifacts/data for this job. inputs: - input: pipelineArtifact - artifactName: MyDrop + artifactName: fluidFrameworkRepo targetPath: ${{ parameters.buildDirectory }} - input: pipelineArtifact artifactName: pack @@ -87,13 +92,6 @@ jobs: runOnce: deploy: steps: - # - task: DownloadPipelineArtifact@2 - # inputs: - # buildType: 'current' - # targetPath: '$(Pipeline.Workspace)' - # artifactName: pack - # - download: current - # artifact: pack - template: /tools/pipelines/templates/include-use-node-version.yml@self - template: /tools/pipelines/templates/include-install-build-tools.yml@self parameters: From 3a844819d0970619a04041b9d54caf39a1426acf Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 20:17:52 +0000 Subject: [PATCH 10/58] Adjusts pipeline to install existing build tools pack --- ...include-publish-npm-package-deployment.yml | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 832f4bca0ef0..dab354ff8739 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -76,9 +76,19 @@ jobs: isProduction: true # This section downloads initial required artifacts/data for this job. inputs: + # - input: pipelineArtifact + # artifactName: fluidFrameworkRepo + # targetPath: ${{ parameters.buildDirectory }} + - input: pipelineArtifact - artifactName: fluidFrameworkRepo + buildType: specific + project: internal + pipeline: 'Build - build-tools' + buildVersionToDownload: latestFromBranch + branchName: ${{ variables['Build.SourceBranch'] }} + artifact: pack targetPath: ${{ parameters.buildDirectory }} + - input: pipelineArtifact artifactName: pack buildType: current @@ -93,11 +103,32 @@ jobs: deploy: steps: - template: /tools/pipelines/templates/include-use-node-version.yml@self - - template: /tools/pipelines/templates/include-install-build-tools.yml@self - parameters: - buildDirectory: ${{ parameters.buildDirectory }} - buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} - pnpmStorePath: ${{ parameters.pnpmStorePath }} + - task: Bash@3 + displayName: List Files in Directory + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + echo "Listing files in directory: $(pwd)" + ls -la + - task: Bash@3 + name: InstallBuildToolsFromTarball + displayName: Install Fluid Build Tools from artifact tarball + inputs: + targetType: 'inline' + workingDirectory: '${{ parameters.buildDirectory }}/tarballs' + script: | + echo "Listing files in directory: $(pwd)" + ls -la + echo "attempting tarball install" + npm i -g fluid-tools-build-cli-*.tgz + + # - template: /tools/pipelines/templates/include-install-build-tools.yml@self + # parameters: + # buildDirectory: ${{ parameters.buildDirectory }} + # buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} + # pnpmStorePath: ${{ parameters.pnpmStorePath }} + - template: /tools/pipelines/templates/include-publish-npm-package-steps.yml@self parameters: artifactPath: tarballs From 6fe99d36b990d9227b5d7fbe38da653a21845d64 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 20:26:34 +0000 Subject: [PATCH 11/58] attempt to fix artifact download issue with 'pipeline' shorthand usage --- .../templates/include-publish-npm-package-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index dab354ff8739..f04936075238 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -81,7 +81,6 @@ jobs: # targetPath: ${{ parameters.buildDirectory }} - input: pipelineArtifact - buildType: specific project: internal pipeline: 'Build - build-tools' buildVersionToDownload: latestFromBranch @@ -111,6 +110,7 @@ jobs: script: | echo "Listing files in directory: $(pwd)" ls -la + - task: Bash@3 name: InstallBuildToolsFromTarball displayName: Install Fluid Build Tools from artifact tarball From b529653de7f583a804ffa41756463e2cfe25c7c1 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 20:35:51 +0000 Subject: [PATCH 12/58] attempt 2 to fix artifact download issue with 'pipeline' shorthand usage --- .../templates/include-publish-npm-package-deployment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index f04936075238..9e24cf949954 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -81,8 +81,9 @@ jobs: # targetPath: ${{ parameters.buildDirectory }} - input: pipelineArtifact + buildType: specific project: internal - pipeline: 'Build - build-tools' + pipeline: Build - build-tools buildVersionToDownload: latestFromBranch branchName: ${{ variables['Build.SourceBranch'] }} artifact: pack From 47fa883c60e7f60d3b0d682b4304d71543ed69f6 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 20:46:57 +0000 Subject: [PATCH 13/58] attempt 3 - change pipeline variable to definition --- .../templates/include-publish-npm-package-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 9e24cf949954..6304d682c23c 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -83,7 +83,7 @@ jobs: - input: pipelineArtifact buildType: specific project: internal - pipeline: Build - build-tools + definition: 14 buildVersionToDownload: latestFromBranch branchName: ${{ variables['Build.SourceBranch'] }} artifact: pack From f6cd7c38833c0dc6d2921812f5db1b3b81120a52 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 20:48:11 +0000 Subject: [PATCH 14/58] syntax fix --- .../templates/include-publish-npm-package-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 6304d682c23c..08f2f00f8c43 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -86,7 +86,7 @@ jobs: definition: 14 buildVersionToDownload: latestFromBranch branchName: ${{ variables['Build.SourceBranch'] }} - artifact: pack + artifactName: pack targetPath: ${{ parameters.buildDirectory }} - input: pipelineArtifact From 01b01a377f08cede1678ad9804083427cbcd8f6b Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 21:42:53 +0000 Subject: [PATCH 15/58] commenting out test steps to speed up pipeline change tests --- tools/pipelines/build-client.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/pipelines/build-client.yml b/tools/pipelines/build-client.yml index db411edda75f..4af158ab9d07 100644 --- a/tools/pipelines/build-client.yml +++ b/tools/pipelines/build-client.yml @@ -206,13 +206,13 @@ extends: # This check must be run after the build, since it relies on built files being present. Eventually it might be moved # to the "pack" stage since it can use the already-packed packages in that case. As it is the pipeline packs some # packages twice. - - { name: "ci:test:jest", jobName: "JestTest" } - - { name: "ci:test:realsvc:tinylicious", jobName: "RealsvcTinyliciousTest" } - - { name: "ci:test:stress:tinylicious", jobName: "StressTinyliciousTest" } - - { name: "check:are-the-types-wrong", jobName: "AreTheTypesWrong" } + # - { name: "ci:test:jest", jobName: "JestTest" } + # - { name: "ci:test:realsvc:tinylicious", jobName: "RealsvcTinyliciousTest" } + # - { name: "ci:test:stress:tinylicious", jobName: "StressTinyliciousTest" } + # - { name: "check:are-the-types-wrong", jobName: "AreTheTypesWrong" } coverageTests: - - { name: "ci:test:mocha", jobName: "MochaTest" } - - { name: "ci:test:realsvc:local", jobName: "RealsvcLocalTest" } + # - { name: "ci:test:mocha", jobName: "MochaTest" } + # - { name: "ci:test:realsvc:local", jobName: "RealsvcLocalTest" } testResultDirs: - nyc/examples - nyc/experimental From f140cb2dfb7831cf3b0d4fb6d78da393ad1c2073 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 21:44:00 +0000 Subject: [PATCH 16/58] comment out test syntax fix --- tools/pipelines/build-client.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/pipelines/build-client.yml b/tools/pipelines/build-client.yml index 4af158ab9d07..c348a952d0b0 100644 --- a/tools/pipelines/build-client.yml +++ b/tools/pipelines/build-client.yml @@ -202,7 +202,7 @@ extends: # We only care about pipeline run telemetry for the CI runs on the internal project, not for PR runs in the public # one. And since we don't batch commits for CI pipeline runs, the only reason we see for automated runs is IndividualCI. telemetry: ${{ eq(variables['Build.Reason'], 'IndividualCI') }} - taskTest: + # taskTest: # This check must be run after the build, since it relies on built files being present. Eventually it might be moved # to the "pack" stage since it can use the already-packed packages in that case. As it is the pipeline packs some # packages twice. @@ -210,7 +210,7 @@ extends: # - { name: "ci:test:realsvc:tinylicious", jobName: "RealsvcTinyliciousTest" } # - { name: "ci:test:stress:tinylicious", jobName: "StressTinyliciousTest" } # - { name: "check:are-the-types-wrong", jobName: "AreTheTypesWrong" } - coverageTests: + # coverageTests: # - { name: "ci:test:mocha", jobName: "MochaTest" } # - { name: "ci:test:realsvc:local", jobName: "RealsvcLocalTest" } testResultDirs: From 71df921f190da94503951356bb98a93af5916129 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 21:46:25 +0000 Subject: [PATCH 17/58] uncomment taskTest params --- tools/pipelines/build-client.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/pipelines/build-client.yml b/tools/pipelines/build-client.yml index c348a952d0b0..2177e119b308 100644 --- a/tools/pipelines/build-client.yml +++ b/tools/pipelines/build-client.yml @@ -202,16 +202,16 @@ extends: # We only care about pipeline run telemetry for the CI runs on the internal project, not for PR runs in the public # one. And since we don't batch commits for CI pipeline runs, the only reason we see for automated runs is IndividualCI. telemetry: ${{ eq(variables['Build.Reason'], 'IndividualCI') }} - # taskTest: + taskTest: # This check must be run after the build, since it relies on built files being present. Eventually it might be moved # to the "pack" stage since it can use the already-packed packages in that case. As it is the pipeline packs some # packages twice. - # - { name: "ci:test:jest", jobName: "JestTest" } + - { name: "ci:test:jest", jobName: "JestTest" } # - { name: "ci:test:realsvc:tinylicious", jobName: "RealsvcTinyliciousTest" } # - { name: "ci:test:stress:tinylicious", jobName: "StressTinyliciousTest" } # - { name: "check:are-the-types-wrong", jobName: "AreTheTypesWrong" } - # coverageTests: - # - { name: "ci:test:mocha", jobName: "MochaTest" } + coverageTests: + - { name: "ci:test:mocha", jobName: "MochaTest" } # - { name: "ci:test:realsvc:local", jobName: "RealsvcLocalTest" } testResultDirs: - nyc/examples From 24b4e39d74319527e078d08b852473b05be2a177 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 22:52:56 +0000 Subject: [PATCH 18/58] removing test steps to speed up pipeline testing and chaning branch requirement --- .../pipelines/templates/build-npm-package.yml | 210 +++++++++--------- ...include-publish-npm-package-deployment.yml | 34 +-- 2 files changed, 122 insertions(+), 122 deletions(-) diff --git a/tools/pipelines/templates/build-npm-package.yml b/tools/pipelines/templates/build-npm-package.yml index c308fe9dfc14..8c1362e53381 100644 --- a/tools/pipelines/templates/build-npm-package.yml +++ b/tools/pipelines/templates/build-npm-package.yml @@ -342,111 +342,111 @@ extends: buildDirectory: '${{ parameters.buildDirectory }}' # Test - - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: - # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not - - script: | - echo "##vso[task.setvariable variable=startTest]true" - displayName: Start Test - - - ${{ each taskTestStep in parameters.taskTest }}: - - template: /tools/pipelines/templates/include-test-task.yml@self - parameters: - taskTestStep: '${{ taskTestStep }}' - buildDirectory: '${{ parameters.buildDirectory }}' - testCoverage: ${{ - and( - parameters.testCoverage, - ne(variables['Build.Reason'], 'PullRequest') - )}} - - - ${{ if contains(convertToJson(parameters.taskTest), 'tinylicious') }}: - - task: Bash@3 - displayName: Upload tinylicious log - condition: always() - continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) - inputs: - targetType: inline - script: | - set -eu -o pipefail - PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; - if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then - echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; - echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; - else - echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; - fi - - # At this point we want to publish the tinylicious.log artifact, but as part of 1ES migration - # that is now part of templateContext.outputs below. - - # Test - Upload coverage results - # Some webpacked file using externals introduce file name with quotes in them - # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish - # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) - - ${{ if parameters.testCoverage }}: - - task: Bash@3 - displayName: Check for nyc/report directory - condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) - inputs: - targetType: 'inline' - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' - - task: Bash@3 - displayName: Patch Coverage Results - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - targetType: 'inline' - workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' - script: | - set -eu -o pipefail - sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml - - task: PublishCodeCoverageResults@2 - displayName: Publish Code Coverage - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml - failIfCoverageEmpty: true - - task: CopyFiles@2 - displayName: Copy code coverage report to artifact staging directory - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' - targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - - task: Bash@3 - displayName: Report Code Coverage Comparison - condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) - continueOnError: false - env: - ADO_API_TOKEN: '$(System.AccessToken)' - GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' - TARGET_BRANCH_NAME: '$(targetBranchName)' - ADO_BUILD_ID: '$(Build.BuildId)' - GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' - GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' - ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 - ADO_CI_BUILD_DEFINITION_ID_PR: 11 - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" - echo "Github PR number: $GITHUB_PR_NUMBER" - echo "ADO Build Number: $ADO_BUILD_ID" - echo "Target Branch Name: $TARGET_BRANCH_NAME" - echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" - echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" - echo "Running code coverage comparison" - flub report codeCoverage --verbose - - - # Process test result, include publishing and logging - - template: /tools/pipelines/templates/include-process-test-results.yml@self - parameters: - buildDirectory: '${{ parameters.buildDirectory }}' - testResultDirs: '${{ parameters.testResultDirs }}' + # - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: + # # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not + # - script: | + # echo "##vso[task.setvariable variable=startTest]true" + # displayName: Start Test + + # - ${{ each taskTestStep in parameters.taskTest }}: + # - template: /tools/pipelines/templates/include-test-task.yml@self + # parameters: + # taskTestStep: '${{ taskTestStep }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + # testCoverage: ${{ + # and( + # parameters.testCoverage, + # ne(variables['Build.Reason'], 'PullRequest') + # )}} + + # - ${{ if contains(convertToJson(parameters.taskTest), 'tinylicious') }}: + # - task: Bash@3 + # displayName: Upload tinylicious log + # condition: always() + # continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) + # inputs: + # targetType: inline + # script: | + # set -eu -o pipefail + # PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; + # if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then + # echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; + # echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; + # else + # echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; + # fi + + # # At this point we want to publish the tinylicious.log artifact, but as part of 1ES migration + # # that is now part of templateContext.outputs below. + + # # Test - Upload coverage results + # # Some webpacked file using externals introduce file name with quotes in them + # # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish + # # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) + # - ${{ if parameters.testCoverage }}: + # - task: Bash@3 + # displayName: Check for nyc/report directory + # condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) + # inputs: + # targetType: 'inline' + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' + # - task: Bash@3 + # displayName: Patch Coverage Results + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # targetType: 'inline' + # workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' + # script: | + # set -eu -o pipefail + # sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml + # - task: PublishCodeCoverageResults@2 + # displayName: Publish Code Coverage + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml + # failIfCoverageEmpty: true + # - task: CopyFiles@2 + # displayName: Copy code coverage report to artifact staging directory + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' + # targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + # - task: Bash@3 + # displayName: Report Code Coverage Comparison + # condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) + # continueOnError: false + # env: + # ADO_API_TOKEN: '$(System.AccessToken)' + # GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' + # TARGET_BRANCH_NAME: '$(targetBranchName)' + # ADO_BUILD_ID: '$(Build.BuildId)' + # GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' + # GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' + # ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 + # ADO_CI_BUILD_DEFINITION_ID_PR: 11 + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" + # echo "Github PR number: $GITHUB_PR_NUMBER" + # echo "ADO Build Number: $ADO_BUILD_ID" + # echo "Target Branch Name: $TARGET_BRANCH_NAME" + # echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" + # echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" + # echo "Running code coverage comparison" + # flub report codeCoverage --verbose + + + # # Process test result, include publishing and logging + # - template: /tools/pipelines/templates/include-process-test-results.yml@self + # parameters: + # buildDirectory: '${{ parameters.buildDirectory }}' + # testResultDirs: '${{ parameters.testResultDirs }}' # Pack - ${{ if ne(parameters.taskPack, false) }}: diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 08f2f00f8c43..823cd69102a9 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -46,22 +46,22 @@ jobs: # This job is necessary to be compliant with 1ES. # 1ES does not allow checking out a code repo within a deployment job so instead we use this pattern # of checking out the repo in separate job and uploading it as a pipeline artifact. -- job: checkoutAndUploadThisRepoAsArtifact - steps: - # Checkout the repository - - checkout: self - persistCredentials: true # Necessary for creation of git tags to work - # Copy required files to the artifact staging directory - - task: CopyFiles@2 - inputs: - SourceFolder: $(System.DefaultWorkingDirectory) - Contents: '**/*' - TargetFolder: $(Build.ArtifactStagingDirectory) - # Publish the artifact containing the required files from the repository - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: $(Build.ArtifactStagingDirectory) - artifactName: fluidFrameworkRepo +# - job: checkoutAndUploadThisRepoAsArtifact +# steps: +# # Checkout the repository +# - checkout: self +# persistCredentials: true # Necessary for creation of git tags to work +# # Copy required files to the artifact staging directory +# - task: CopyFiles@2 +# inputs: +# SourceFolder: $(System.DefaultWorkingDirectory) +# Contents: '**/*' +# TargetFolder: $(Build.ArtifactStagingDirectory) +# # Publish the artifact containing the required files from the repository +# - task: 1ES.PublishPipelineArtifact@1 +# inputs: +# targetPath: $(Build.ArtifactStagingDirectory) +# artifactName: fluidFrameworkRepo - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job @@ -85,7 +85,7 @@ jobs: project: internal definition: 14 buildVersionToDownload: latestFromBranch - branchName: ${{ variables['Build.SourceBranch'] }} + branchName: main artifactName: pack targetPath: ${{ parameters.buildDirectory }} From 33b81a4377ad81f87adbe463fb3c82e5d7527410 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 22:54:06 +0000 Subject: [PATCH 19/58] remove dependsOn --- .../templates/include-publish-npm-package-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 823cd69102a9..b442d7cd47a6 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -65,7 +65,7 @@ jobs: - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job - dependsOn: checkoutAndUploadThisRepoAsArtifact + # dependsOn: checkoutAndUploadThisRepoAsArtifact displayName: Publish ${{ parameters.environment }} pool: ${{ parameters.pool }} environment: ${{ parameters.environment }} From a0b54e4910d3420dcba3406cb3442fad8e1222f1 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 23:49:44 +0000 Subject: [PATCH 20/58] swapping input download to alternate format --- .../pipelines/templates/build-npm-package.yml | 212 +++++++++--------- ...include-publish-npm-package-deployment.yml | 4 +- 2 files changed, 107 insertions(+), 109 deletions(-) diff --git a/tools/pipelines/templates/build-npm-package.yml b/tools/pipelines/templates/build-npm-package.yml index 8c1362e53381..dc6bbc48e6aa 100644 --- a/tools/pipelines/templates/build-npm-package.yml +++ b/tools/pipelines/templates/build-npm-package.yml @@ -341,112 +341,112 @@ extends: taskLintName: '${{ parameters.taskLintName }}' buildDirectory: '${{ parameters.buildDirectory }}' - # Test - # - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: - # # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not - # - script: | - # echo "##vso[task.setvariable variable=startTest]true" - # displayName: Start Test - - # - ${{ each taskTestStep in parameters.taskTest }}: - # - template: /tools/pipelines/templates/include-test-task.yml@self - # parameters: - # taskTestStep: '${{ taskTestStep }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - # testCoverage: ${{ - # and( - # parameters.testCoverage, - # ne(variables['Build.Reason'], 'PullRequest') - # )}} - - # - ${{ if contains(convertToJson(parameters.taskTest), 'tinylicious') }}: - # - task: Bash@3 - # displayName: Upload tinylicious log - # condition: always() - # continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) - # inputs: - # targetType: inline - # script: | - # set -eu -o pipefail - # PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; - # if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then - # echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; - # echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; - # else - # echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; - # fi - - # # At this point we want to publish the tinylicious.log artifact, but as part of 1ES migration - # # that is now part of templateContext.outputs below. - - # # Test - Upload coverage results - # # Some webpacked file using externals introduce file name with quotes in them - # # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish - # # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) - # - ${{ if parameters.testCoverage }}: - # - task: Bash@3 - # displayName: Check for nyc/report directory - # condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) - # inputs: - # targetType: 'inline' - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' - # - task: Bash@3 - # displayName: Patch Coverage Results - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # targetType: 'inline' - # workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' - # script: | - # set -eu -o pipefail - # sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml - # - task: PublishCodeCoverageResults@2 - # displayName: Publish Code Coverage - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml - # failIfCoverageEmpty: true - # - task: CopyFiles@2 - # displayName: Copy code coverage report to artifact staging directory - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' - # targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - # - task: Bash@3 - # displayName: Report Code Coverage Comparison - # condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) - # continueOnError: false - # env: - # ADO_API_TOKEN: '$(System.AccessToken)' - # GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' - # TARGET_BRANCH_NAME: '$(targetBranchName)' - # ADO_BUILD_ID: '$(Build.BuildId)' - # GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' - # GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' - # ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 - # ADO_CI_BUILD_DEFINITION_ID_PR: 11 - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" - # echo "Github PR number: $GITHUB_PR_NUMBER" - # echo "ADO Build Number: $ADO_BUILD_ID" - # echo "Target Branch Name: $TARGET_BRANCH_NAME" - # echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" - # echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" - # echo "Running code coverage comparison" - # flub report codeCoverage --verbose - - - # # Process test result, include publishing and logging - # - template: /tools/pipelines/templates/include-process-test-results.yml@self - # parameters: - # buildDirectory: '${{ parameters.buildDirectory }}' - # testResultDirs: '${{ parameters.testResultDirs }}' + #Test + - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: + # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not + - script: | + echo "##vso[task.setvariable variable=startTest]true" + displayName: Start Test + + - ${{ each taskTestStep in parameters.taskTest }}: + - template: /tools/pipelines/templates/include-test-task.yml@self + parameters: + taskTestStep: '${{ taskTestStep }}' + buildDirectory: '${{ parameters.buildDirectory }}' + testCoverage: ${{ + and( + parameters.testCoverage, + ne(variables['Build.Reason'], 'PullRequest') + )}} + + - ${{ if contains(convertToJson(parameters.taskTest), 'tinylicious') }}: + - task: Bash@3 + displayName: Upload tinylicious log + condition: always() + continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) + inputs: + targetType: inline + script: | + set -eu -o pipefail + PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; + if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then + echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; + echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; + else + echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; + fi + + # At this point we want to publish the tinylicious.log artifact, but as part of 1ES migration + # that is now part of templateContext.outputs below. + + # Test - Upload coverage results + # Some webpacked file using externals introduce file name with quotes in them + # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish + # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) + - ${{ if parameters.testCoverage }}: + - task: Bash@3 + displayName: Check for nyc/report directory + condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) + inputs: + targetType: 'inline' + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' + - task: Bash@3 + displayName: Patch Coverage Results + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + targetType: 'inline' + workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' + script: | + set -eu -o pipefail + sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml + - task: PublishCodeCoverageResults@2 + displayName: Publish Code Coverage + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml + failIfCoverageEmpty: true + - task: CopyFiles@2 + displayName: Copy code coverage report to artifact staging directory + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' + targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + - task: Bash@3 + displayName: Report Code Coverage Comparison + condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) + continueOnError: false + env: + ADO_API_TOKEN: '$(System.AccessToken)' + GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' + TARGET_BRANCH_NAME: '$(targetBranchName)' + ADO_BUILD_ID: '$(Build.BuildId)' + GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' + GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' + ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 + ADO_CI_BUILD_DEFINITION_ID_PR: 11 + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" + echo "Github PR number: $GITHUB_PR_NUMBER" + echo "ADO Build Number: $ADO_BUILD_ID" + echo "Target Branch Name: $TARGET_BRANCH_NAME" + echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" + echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" + echo "Running code coverage comparison" + flub report codeCoverage --verbose + + + # Process test result, include publishing and logging + - template: /tools/pipelines/templates/include-process-test-results.yml@self + parameters: + buildDirectory: '${{ parameters.buildDirectory }}' + testResultDirs: '${{ parameters.testResultDirs }}' # Pack - ${{ if ne(parameters.taskPack, false) }}: diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index b442d7cd47a6..d0caf6bd4c2e 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -82,9 +82,7 @@ jobs: - input: pipelineArtifact buildType: specific - project: internal - definition: 14 - buildVersionToDownload: latestFromBranch + pipeline: 'Build - build-tools' branchName: main artifactName: pack targetPath: ${{ parameters.buildDirectory }} From 61fbf859691a7cf370d49232d06cae761cb8e518 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 25 Feb 2025 23:55:07 +0000 Subject: [PATCH 21/58] 1 more change to input artifact syntax --- .../templates/include-publish-npm-package-deployment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index d0caf6bd4c2e..55b50010fd13 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -81,7 +81,6 @@ jobs: # targetPath: ${{ parameters.buildDirectory }} - input: pipelineArtifact - buildType: specific pipeline: 'Build - build-tools' branchName: main artifactName: pack From 74da2486e3157a64c08c57f2f252c48284d92f6f Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:21:26 +0000 Subject: [PATCH 22/58] injecting artifact download step early to confirm it works --- .../pipelines/templates/build-npm-client-package.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 57d1f7d2ed11..7b871eb8d362 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -216,6 +216,17 @@ extends: - name: targetBranchName value: $(System.PullRequest.TargetBranch) steps: + # test step just to see if we can download the artifact as expected + - task: DownloadPipelineArtifact@2 + inputs: + buildType: specific + project: internal + pipeline: 'Build - build-tools' + buildVersionToDownload: latestFromBranch + branchName: main + artifact: pack + allowPartiallySucceededBuilds: true + # Setup - checkout: self clean: true From 501b9ea0443d976c0c64acc47ff5357c6f6a2287 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:54:26 +0000 Subject: [PATCH 23/58] Adds build build tools as resource to build client packages --- tools/pipelines/templates/build-npm-client-package.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 7b871eb8d362..f31c5717f66e 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -147,6 +147,11 @@ resources: type: git name: 1ESPipelineTemplates/M365GPT ref: refs/tags/release + pipelines: + - pipeline: buildTools-resource + project: internal + source: Build - build-tools + branch: main extends: # The pipeline extends the 1ES pipeline template which will inject different SDL and compliance tasks. @@ -225,7 +230,6 @@ extends: buildVersionToDownload: latestFromBranch branchName: main artifact: pack - allowPartiallySucceededBuilds: true # Setup - checkout: self From f5f02f367e265eab796813775e2b8ff10a7e313c Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Wed, 26 Feb 2025 19:01:59 +0000 Subject: [PATCH 24/58] attempt to list auto downloaded files from pipeline resource --- .../templates/build-npm-client-package.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index f31c5717f66e..f236ca645b2d 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -222,14 +222,14 @@ extends: value: $(System.PullRequest.TargetBranch) steps: # test step just to see if we can download the artifact as expected - - task: DownloadPipelineArtifact@2 + - task: Bash@3 + displayName: List Files in Directory inputs: - buildType: specific - project: internal - pipeline: 'Build - build-tools' - buildVersionToDownload: latestFromBranch - branchName: main - artifact: pack + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + echo "Listing files in directory: $(PIPELINE.WORKSPACE)/buildTools-resource/pack" + ls -la "$(PIPELINE.WORKSPACE)/buildTools-resource/pack" # Setup - checkout: self From 351ff8b8a2e77a41756dbaeab830d1f7c177ef7f Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Wed, 26 Feb 2025 19:11:50 +0000 Subject: [PATCH 25/58] Just try to find consumed resources in workspace --- tools/pipelines/templates/build-npm-client-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index f236ca645b2d..a430c5ff9ed9 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -223,13 +223,13 @@ extends: steps: # test step just to see if we can download the artifact as expected - task: Bash@3 - displayName: List Files in Directory + displayName: List Files in PIPELINE.WORKSPACE inputs: targetType: inline workingDirectory: '${{ parameters.buildDirectory }}' script: | - echo "Listing files in directory: $(PIPELINE.WORKSPACE)/buildTools-resource/pack" - ls -la "$(PIPELINE.WORKSPACE)/buildTools-resource/pack" + echo "Listing files in directory: $(Pipeline.Workspace)" + ls -la "$(Pipeline.Workspace)/" # Setup - checkout: self From b69fe191e6513251a498856cdbc07ae7e730ca04 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:07:26 +0000 Subject: [PATCH 26/58] adds test deployment job to see if consumed pipeline variables are available --- .../templates/build-npm-client-package.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index a430c5ff9ed9..dae1fa034663 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -192,11 +192,33 @@ extends: # Install all dependencies, not just the root ones dependencyInstallCommand: pnpm install --frozen-lockfile + # Install / Build / Test Stage - stage: build displayName: Build Stage dependsOn: [] # this stage doesn't depend on preceding stage jobs: + - deployment: test deployment job + displayName: 'test deployment job' + environment: 'fluid-docs-env' + pool: Large-eastus2 + strategy: + runOnce: + deploy: + steps: + - checkout: none + clean: true + + # test step just to see if we can download the artifact as expected + - task: Bash@3 + displayName: List Files in PIPELINE.WORKSPACE + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + echo "Listing files in directory: $(Pipeline.Workspace)" + ls -la "$(Pipeline.Workspace)/" + # Job - Build - job: build displayName: Build From 9b10631467055b1870708636f389d275f449995d Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:09:21 +0000 Subject: [PATCH 27/58] makes deployment job name one word --- tools/pipelines/templates/build-npm-client-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index dae1fa034663..c613630edc2c 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -198,7 +198,7 @@ extends: displayName: Build Stage dependsOn: [] # this stage doesn't depend on preceding stage jobs: - - deployment: test deployment job + - deployment: testDeploymentJob displayName: 'test deployment job' environment: 'fluid-docs-env' pool: Large-eastus2 From c11ec4d725c2c92dabb5524fcbdaa9e2468a2696 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:13:53 +0000 Subject: [PATCH 28/58] attempt to install build tools from tarball --- .../pipelines/templates/build-npm-client-package.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index c613630edc2c..d099f4dd8bb0 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -219,6 +219,18 @@ extends: echo "Listing files in directory: $(Pipeline.Workspace)" ls -la "$(Pipeline.Workspace)/" + - task: Bash@3 + name: InstallBuildToolsFromTarball + displayName: Install Fluid Build Tools from artifact tarball + inputs: + targetType: 'inline' + workingDirectory: '"$(Pipeline.Workspace)/buildTools-resource/tarballs' + script: | + echo "Listing files in directory: $(pwd)" + ls -la + echo "attempting tarball install" + npm i -g fluid-tools-build-cli-*.tgz + # Job - Build - job: build displayName: Build From 90a5314c4c745ad9ffe30d4d911ebbb86d100362 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:19:05 +0000 Subject: [PATCH 29/58] fixes syntax issue --- tools/pipelines/templates/build-npm-client-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index d099f4dd8bb0..8ceb68da529a 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -224,7 +224,7 @@ extends: displayName: Install Fluid Build Tools from artifact tarball inputs: targetType: 'inline' - workingDirectory: '"$(Pipeline.Workspace)/buildTools-resource/tarballs' + workingDirectory: '$(Pipeline.Workspace)/buildTools-resource/tarballs' script: | echo "Listing files in directory: $(pwd)" ls -la From f56f05f3278523e2b1a91765edf9bdb0427e6cd7 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:28:41 +0000 Subject: [PATCH 30/58] updated step to list contents of buildTools-resoruce folder --- .../templates/include-publish-npm-package-deployment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 55b50010fd13..53dde8732283 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -108,6 +108,8 @@ jobs: script: | echo "Listing files in directory: $(pwd)" ls -la + cd ./buildTools-resource + ls -la - task: Bash@3 name: InstallBuildToolsFromTarball From 2aea7ad5f3c88bf5627d4fe5e62248fe18838fdd Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:30:42 +0000 Subject: [PATCH 31/58] Adding last change to build package pipeline --- tools/pipelines/templates/build-npm-client-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 8ceb68da529a..f411ea34073f 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -206,9 +206,6 @@ extends: runOnce: deploy: steps: - - checkout: none - clean: true - # test step just to see if we can download the artifact as expected - task: Bash@3 displayName: List Files in PIPELINE.WORKSPACE @@ -218,6 +215,9 @@ extends: script: | echo "Listing files in directory: $(Pipeline.Workspace)" ls -la "$(Pipeline.Workspace)/" + cd ./buildTools-resource + echo "Listing files in directory: $(Pipeline.Workspace)/buildTools-resource" + ls -la - task: Bash@3 name: InstallBuildToolsFromTarball From b8758fe045d3efab541eca3e974ead971c6d6f59 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:31:37 +0000 Subject: [PATCH 32/58] environemnt update for test deployment job --- tools/pipelines/templates/build-npm-client-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index f411ea34073f..e9c2644d5ddf 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -200,7 +200,7 @@ extends: jobs: - deployment: testDeploymentJob displayName: 'test deployment job' - environment: 'fluid-docs-env' + environment: 'test-package-build-feed' pool: Large-eastus2 strategy: runOnce: From 09b524d7f3803c742df7a8a8f4bd5c7c69abf72d Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:33:30 +0000 Subject: [PATCH 33/58] revert to env name --- tools/pipelines/templates/build-npm-client-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index e9c2644d5ddf..f411ea34073f 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -200,7 +200,7 @@ extends: jobs: - deployment: testDeploymentJob displayName: 'test deployment job' - environment: 'test-package-build-feed' + environment: 'fluid-docs-env' pool: Large-eastus2 strategy: runOnce: From 050e4f46a49c693588c638b67e495c13561cc623 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:39:57 +0000 Subject: [PATCH 34/58] minor adjustments to folder paths --- tools/pipelines/templates/build-npm-client-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index f411ea34073f..32d907b33e02 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -215,7 +215,7 @@ extends: script: | echo "Listing files in directory: $(Pipeline.Workspace)" ls -la "$(Pipeline.Workspace)/" - cd ./buildTools-resource + cd $(Pipeline.Workspace)/buildTools-resource echo "Listing files in directory: $(Pipeline.Workspace)/buildTools-resource" ls -la @@ -224,7 +224,7 @@ extends: displayName: Install Fluid Build Tools from artifact tarball inputs: targetType: 'inline' - workingDirectory: '$(Pipeline.Workspace)/buildTools-resource/tarballs' + workingDirectory: '$(Pipeline.Workspace)/buildTools-resource/pack/tarballs' script: | echo "Listing files in directory: $(pwd)" ls -la From 4c7aeec070c79423ecf81758b4a579c68e36db5f Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:52:55 +0000 Subject: [PATCH 35/58] adjust tarball install to install all tarballs at the same time --- tools/pipelines/templates/build-npm-client-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 32d907b33e02..25cd641ad4c7 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -229,7 +229,7 @@ extends: echo "Listing files in directory: $(pwd)" ls -la echo "attempting tarball install" - npm i -g fluid-tools-build-cli-*.tgz + npm i -g ./*.tgz # Job - Build - job: build From bbae6464ea20ee59212ea5a14ab130d05adca613 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 28 Feb 2025 16:24:44 +0000 Subject: [PATCH 36/58] Moves new logic to publish npm package deployment and removes changes from other pipelines --- .../templates/build-npm-client-package.yml | 43 ---------- .../pipelines/templates/build-npm-package.yml | 2 +- ...include-publish-npm-package-deployment.yml | 79 ++++++------------- 3 files changed, 24 insertions(+), 100 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 25cd641ad4c7..6c9798b28fe2 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -198,39 +198,6 @@ extends: displayName: Build Stage dependsOn: [] # this stage doesn't depend on preceding stage jobs: - - deployment: testDeploymentJob - displayName: 'test deployment job' - environment: 'fluid-docs-env' - pool: Large-eastus2 - strategy: - runOnce: - deploy: - steps: - # test step just to see if we can download the artifact as expected - - task: Bash@3 - displayName: List Files in PIPELINE.WORKSPACE - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - echo "Listing files in directory: $(Pipeline.Workspace)" - ls -la "$(Pipeline.Workspace)/" - cd $(Pipeline.Workspace)/buildTools-resource - echo "Listing files in directory: $(Pipeline.Workspace)/buildTools-resource" - ls -la - - - task: Bash@3 - name: InstallBuildToolsFromTarball - displayName: Install Fluid Build Tools from artifact tarball - inputs: - targetType: 'inline' - workingDirectory: '$(Pipeline.Workspace)/buildTools-resource/pack/tarballs' - script: | - echo "Listing files in directory: $(pwd)" - ls -la - echo "attempting tarball install" - npm i -g ./*.tgz - # Job - Build - job: build displayName: Build @@ -255,16 +222,6 @@ extends: - name: targetBranchName value: $(System.PullRequest.TargetBranch) steps: - # test step just to see if we can download the artifact as expected - - task: Bash@3 - displayName: List Files in PIPELINE.WORKSPACE - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - echo "Listing files in directory: $(Pipeline.Workspace)" - ls -la "$(Pipeline.Workspace)/" - # Setup - checkout: self clean: true diff --git a/tools/pipelines/templates/build-npm-package.yml b/tools/pipelines/templates/build-npm-package.yml index dc6bbc48e6aa..c308fe9dfc14 100644 --- a/tools/pipelines/templates/build-npm-package.yml +++ b/tools/pipelines/templates/build-npm-package.yml @@ -341,7 +341,7 @@ extends: taskLintName: '${{ parameters.taskLintName }}' buildDirectory: '${{ parameters.buildDirectory }}' - #Test + # Test - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not - script: | diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 53dde8732283..ff28aeff2593 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -41,28 +41,15 @@ parameters: type: string default: $(Pipeline.Workspace)/.pnpm-store -jobs: - - # This job is necessary to be compliant with 1ES. - # 1ES does not allow checking out a code repo within a deployment job so instead we use this pattern - # of checking out the repo in separate job and uploading it as a pipeline artifact. -# - job: checkoutAndUploadThisRepoAsArtifact -# steps: -# # Checkout the repository -# - checkout: self -# persistCredentials: true # Necessary for creation of git tags to work -# # Copy required files to the artifact staging directory -# - task: CopyFiles@2 -# inputs: -# SourceFolder: $(System.DefaultWorkingDirectory) -# Contents: '**/*' -# TargetFolder: $(Build.ArtifactStagingDirectory) -# # Publish the artifact containing the required files from the repository -# - task: 1ES.PublishPipelineArtifact@1 -# inputs: -# targetPath: $(Build.ArtifactStagingDirectory) -# artifactName: fluidFrameworkRepo +resources: + # pipeline listed as a resource have their artifacts automatically downloeded in a deployment job. + pipelines: + - pipeline: buildTools-resource + project: internal + source: Build - build-tools + branch: main +jobs: - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job # dependsOn: checkoutAndUploadThisRepoAsArtifact @@ -74,18 +61,8 @@ jobs: templateContext: type: releaseJob isProduction: true - # This section downloads initial required artifacts/data for this job. + # Inputs here are automatically downloaded at the beginning of the job. inputs: - # - input: pipelineArtifact - # artifactName: fluidFrameworkRepo - # targetPath: ${{ parameters.buildDirectory }} - - - input: pipelineArtifact - pipeline: 'Build - build-tools' - branchName: main - artifactName: pack - targetPath: ${{ parameters.buildDirectory }} - - input: pipelineArtifact artifactName: pack buildType: current @@ -100,35 +77,17 @@ jobs: deploy: steps: - template: /tools/pipelines/templates/include-use-node-version.yml@self - - task: Bash@3 - displayName: List Files in Directory - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - echo "Listing files in directory: $(pwd)" - ls -la - cd ./buildTools-resource - ls -la - - task: Bash@3 name: InstallBuildToolsFromTarball - displayName: Install Fluid Build Tools from artifact tarball + displayName: Install Fluid Build Tools from Artifact Tarball inputs: targetType: 'inline' - workingDirectory: '${{ parameters.buildDirectory }}/tarballs' + workingDirectory: '$(Pipeline.Workspace)/buildTools-resource/pack/tarballs' script: | echo "Listing files in directory: $(pwd)" ls -la echo "attempting tarball install" - npm i -g fluid-tools-build-cli-*.tgz - - # - template: /tools/pipelines/templates/include-install-build-tools.yml@self - # parameters: - # buildDirectory: ${{ parameters.buildDirectory }} - # buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} - # pnpmStorePath: ${{ parameters.pnpmStorePath }} - + npm i -g ./*.tgz - template: /tools/pipelines/templates/include-publish-npm-package-steps.yml@self parameters: artifactPath: tarballs @@ -136,6 +95,14 @@ jobs: customEndPoint: ${{ parameters.customEndPoint }} feedKind: ${{ parameters.feedKind }} publishFlags: ${{ parameters.publishFlags }} - - template: /tools/pipelines/templates/include-git-tag-steps.yml@self - parameters: - tagName: ${{ parameters.tagName }} + +- job: gitTagForBuild + displayName: Push Git Tag for Build + dependsOn: publish_${{ replace(parameters.environment, '-', '_') }} + steps: + - checkout: self + clean: true + persistCredentials: true # Necessary for creation of git tags to work + - template: /tools/pipelines/templates/include-git-tag-steps.yml@self + parameters: + tagName: ${{ parameters.tagName }} From 9eb0ee85541b4c8581b145241198c63bf4f9989d Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 28 Feb 2025 16:26:32 +0000 Subject: [PATCH 37/58] removes pipeline resource added to build-npm-client-package --- tools/pipelines/templates/build-npm-client-package.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 6c9798b28fe2..57d1f7d2ed11 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -147,11 +147,6 @@ resources: type: git name: 1ESPipelineTemplates/M365GPT ref: refs/tags/release - pipelines: - - pipeline: buildTools-resource - project: internal - source: Build - build-tools - branch: main extends: # The pipeline extends the 1ES pipeline template which will inject different SDL and compliance tasks. @@ -192,7 +187,6 @@ extends: # Install all dependencies, not just the root ones dependencyInstallCommand: pnpm install --frozen-lockfile - # Install / Build / Test Stage - stage: build displayName: Build Stage From a58fd296ac71429d7ad18e4a51602a5193bbc7e3 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 28 Feb 2025 16:30:02 +0000 Subject: [PATCH 38/58] moves around comment location --- .../templates/include-publish-npm-package-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index ff28aeff2593..47ee174ccb11 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -42,8 +42,8 @@ parameters: default: $(Pipeline.Workspace)/.pnpm-store resources: - # pipeline listed as a resource have their artifacts automatically downloeded in a deployment job. pipelines: + # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. - pipeline: buildTools-resource project: internal source: Build - build-tools From 970d7a60d34e6b2d6e738b3209ee86074e26d374 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 28 Feb 2025 16:36:58 +0000 Subject: [PATCH 39/58] Adds build tools pipeline as a resource to build npm client package as well --- tools/pipelines/templates/build-npm-client-package.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 57d1f7d2ed11..7be6a4129379 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -147,6 +147,12 @@ resources: type: git name: 1ESPipelineTemplates/M365GPT ref: refs/tags/release + pipelines: + # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. + - pipeline: buildTools-resource + project: internal + source: Build - build-tools + branch: main extends: # The pipeline extends the 1ES pipeline template which will inject different SDL and compliance tasks. From 3a5ded543e290cacb00add4990887148b33a443f Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 28 Feb 2025 16:39:04 +0000 Subject: [PATCH 40/58] attempt to resolve Unexpected value 'resources' error --- .../templates/include-publish-npm-package-deployment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 47ee174ccb11..359c79eee947 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -43,7 +43,6 @@ parameters: resources: pipelines: - # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. - pipeline: buildTools-resource project: internal source: Build - build-tools From 89d712edd720c587fac35ac9639cc044ca797c49 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 28 Feb 2025 16:39:57 +0000 Subject: [PATCH 41/58] Removes resource section from include-publish-npm-package-deployment --- .../templates/include-publish-npm-package-deployment.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 359c79eee947..8c2152cfeab7 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -41,13 +41,6 @@ parameters: type: string default: $(Pipeline.Workspace)/.pnpm-store -resources: - pipelines: - - pipeline: buildTools-resource - project: internal - source: Build - build-tools - branch: main - jobs: - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job From ab5a2cec41121d937b54a32a81d38f7c823e3d85 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 15:58:02 +0000 Subject: [PATCH 42/58] Moves resource section to include publish npm exclusively --- tools/pipelines/templates/build-npm-client-package.yml | 6 ------ .../templates/include-publish-npm-package-deployment.yml | 8 ++++++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 7be6a4129379..57d1f7d2ed11 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -147,12 +147,6 @@ resources: type: git name: 1ESPipelineTemplates/M365GPT ref: refs/tags/release - pipelines: - # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. - - pipeline: buildTools-resource - project: internal - source: Build - build-tools - branch: main extends: # The pipeline extends the 1ES pipeline template which will inject different SDL and compliance tasks. diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 8c2152cfeab7..47ee174ccb11 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -41,6 +41,14 @@ parameters: type: string default: $(Pipeline.Workspace)/.pnpm-store +resources: + pipelines: + # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. + - pipeline: buildTools-resource + project: internal + source: Build - build-tools + branch: main + jobs: - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job From 4bcd4884661a973bab36cc29751e47f01c5a0b8a Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 16:18:09 +0000 Subject: [PATCH 43/58] Revert "Moves resource section to include publish npm exclusively" This reverts commit ab5a2cec41121d937b54a32a81d38f7c823e3d85. --- tools/pipelines/templates/build-npm-client-package.yml | 6 ++++++ .../templates/include-publish-npm-package-deployment.yml | 8 -------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 57d1f7d2ed11..7be6a4129379 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -147,6 +147,12 @@ resources: type: git name: 1ESPipelineTemplates/M365GPT ref: refs/tags/release + pipelines: + # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. + - pipeline: buildTools-resource + project: internal + source: Build - build-tools + branch: main extends: # The pipeline extends the 1ES pipeline template which will inject different SDL and compliance tasks. diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 47ee174ccb11..8c2152cfeab7 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -41,14 +41,6 @@ parameters: type: string default: $(Pipeline.Workspace)/.pnpm-store -resources: - pipelines: - # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. - - pipeline: buildTools-resource - project: internal - source: Build - build-tools - branch: main - jobs: - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job From bce219013db39dcf2c5ecaf8060c1e408041fd6a Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 16:38:26 +0000 Subject: [PATCH 44/58] comments out build stage to figure out if template stage deployment job has access to downloaded artifacts --- .../pipelines/templates/build-npm-client-package.yml | 2 +- .../include-publish-npm-package-deployment.yml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 7be6a4129379..941dc0d870a5 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -194,7 +194,7 @@ extends: dependencyInstallCommand: pnpm install --frozen-lockfile # Install / Build / Test Stage - - stage: build + # - stage: build displayName: Build Stage dependsOn: [] # this stage doesn't depend on preceding stage jobs: diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 8c2152cfeab7..9c2ce3a348dc 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -69,6 +69,17 @@ jobs: deploy: steps: - template: /tools/pipelines/templates/include-use-node-version.yml@self + + - task: Bash@3 + name: listFilesInWorkspace + displayName: List files in worksspace + inputs: + targetType: 'inline' + workingDirectory: '$(Pipeline.Workspace)' + script: | + echo "Listing files in directory: $(pwd)" + ls -la + - task: Bash@3 name: InstallBuildToolsFromTarball displayName: Install Fluid Build Tools from Artifact Tarball From 25bd4718ff1ef33868b04f0558547dba106b5790 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 16:41:18 +0000 Subject: [PATCH 45/58] moves publish stage to top of build npm client package for testing --- .../templates/build-npm-client-package.yml | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 941dc0d870a5..3ecca02a485f 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -193,8 +193,17 @@ extends: # Install all dependencies, not just the root ones dependencyInstallCommand: pnpm install --frozen-lockfile + # Publish stage + - ${{ if eq(parameters.publish, true) }}: + - template: /tools/pipelines/templates/include-publish-npm-package.yml@self + parameters: + tagName: ${{ parameters.tagName }} + isReleaseGroup: ${{ parameters.isReleaseGroup }} + buildDirectory: ${{ parameters.buildDirectory }} + buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} + # Install / Build / Test Stage - # - stage: build + - stage: build displayName: Build Stage dependsOn: [] # this stage doesn't depend on preceding stage jobs: @@ -795,14 +804,14 @@ extends: buildDirectory: '${{ parameters.buildDirectory }}' testResultDirs: '${{ parameters.testResultDirs }}' - # Publish stage - - ${{ if eq(parameters.publish, true) }}: - - template: /tools/pipelines/templates/include-publish-npm-package.yml@self - parameters: - tagName: ${{ parameters.tagName }} - isReleaseGroup: ${{ parameters.isReleaseGroup }} - buildDirectory: ${{ parameters.buildDirectory }} - buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} + # # Publish stage + # - ${{ if eq(parameters.publish, true) }}: + # - template: /tools/pipelines/templates/include-publish-npm-package.yml@self + # parameters: + # tagName: ${{ parameters.tagName }} + # isReleaseGroup: ${{ parameters.isReleaseGroup }} + # buildDirectory: ${{ parameters.buildDirectory }} + # buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Capture pipeline stage results - ${{ if eq(parameters.telemetry, true) }}: From db4cdeca04bb3fd08fac8c3a64cdc2d4b1bffee7 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 16:54:12 +0000 Subject: [PATCH 46/58] more adjustmnets to get testing to go faster --- .../templates/build-npm-client-package.yml | 28 +++++++------- ...include-publish-npm-package-deployment.yml | 38 ++++++++++--------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 3ecca02a485f..ff19587e5fef 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -185,22 +185,22 @@ extends: customBuildTags: - ES365AIMigrationTooling stages: - - ${{ if ne(convertToJson(parameters.checks), '[]') }}: - - template: /tools/pipelines/templates/include-policy-check.yml@self - parameters: - buildDirectory: '${{ parameters.buildDirectory }}' - checks: '${{ parameters.checks }}' - # Install all dependencies, not just the root ones - dependencyInstallCommand: pnpm install --frozen-lockfile + # - ${{ if ne(convertToJson(parameters.checks), '[]') }}: + # - template: /tools/pipelines/templates/include-policy-check.yml@self + # parameters: + # buildDirectory: '${{ parameters.buildDirectory }}' + # checks: '${{ parameters.checks }}' + # # Install all dependencies, not just the root ones + # dependencyInstallCommand: pnpm install --frozen-lockfile # Publish stage - - ${{ if eq(parameters.publish, true) }}: - - template: /tools/pipelines/templates/include-publish-npm-package.yml@self - parameters: - tagName: ${{ parameters.tagName }} - isReleaseGroup: ${{ parameters.isReleaseGroup }} - buildDirectory: ${{ parameters.buildDirectory }} - buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} + # - ${{ if eq(parameters.publish, true) }}: + - template: /tools/pipelines/templates/include-publish-npm-package.yml@self + parameters: + tagName: ${{ parameters.tagName }} + isReleaseGroup: ${{ parameters.isReleaseGroup }} + buildDirectory: ${{ parameters.buildDirectory }} + buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Install / Build / Test Stage - stage: build diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index 9c2ce3a348dc..e4978c3d5181 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -59,6 +59,7 @@ jobs: artifactName: pack buildType: current targetPath: $(Pipeline.Workspace)/pack + workspace: clean: all variables: @@ -91,21 +92,22 @@ jobs: ls -la echo "attempting tarball install" npm i -g ./*.tgz - - template: /tools/pipelines/templates/include-publish-npm-package-steps.yml@self - parameters: - artifactPath: tarballs - feedUrl: ${{ parameters.feedUrl }} - customEndPoint: ${{ parameters.customEndPoint }} - feedKind: ${{ parameters.feedKind }} - publishFlags: ${{ parameters.publishFlags }} - -- job: gitTagForBuild - displayName: Push Git Tag for Build - dependsOn: publish_${{ replace(parameters.environment, '-', '_') }} - steps: - - checkout: self - clean: true - persistCredentials: true # Necessary for creation of git tags to work - - template: /tools/pipelines/templates/include-git-tag-steps.yml@self - parameters: - tagName: ${{ parameters.tagName }} + + # - template: /tools/pipelines/templates/include-publish-npm-package-steps.yml@self + # parameters: + # artifactPath: tarballs + # feedUrl: ${{ parameters.feedUrl }} + # customEndPoint: ${{ parameters.customEndPoint }} + # feedKind: ${{ parameters.feedKind }} + # publishFlags: ${{ parameters.publishFlags }} + +# - job: gitTagForBuild +# displayName: Push Git Tag for Build +# dependsOn: publish_${{ replace(parameters.environment, '-', '_') }} +# steps: +# - checkout: self +# clean: true +# persistCredentials: true # Necessary for creation of git tags to work +# - template: /tools/pipelines/templates/include-git-tag-steps.yml@self +# parameters: +# tagName: ${{ parameters.tagName }} From 8fa5a92ab6995133a0cb2accb0d2d4d8fa3fd874 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 16:55:15 +0000 Subject: [PATCH 47/58] comments out build stage --- .../templates/build-npm-client-package.yml | 1200 ++++++++--------- 1 file changed, 600 insertions(+), 600 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index ff19587e5fef..710ee8620a7a 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -203,606 +203,606 @@ extends: buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Install / Build / Test Stage - - stage: build - displayName: Build Stage - dependsOn: [] # this stage doesn't depend on preceding stage - jobs: - # Job - Build - - job: build - displayName: Build - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - timeoutInMinutes: 120 - ${{ else }}: - # CI builds run more aggressive compat configurations which can take longer. - # See "FullCompat" under packages\test\test-version-utils\README.md for more details. - # At the time of adding this comment, the full compat config is on the smaller side and so - # CI builds consistently pass with a 60 minutes timeout. However, it will naturally grow - # over time and it might be necessary to bump it. - # AB#6680 is also relevant here, which tracks rethinking how and where we run tests (likely with - # a focus on e2e tests) - # Note, This was recently updated to 90 minutes to account for the additional build time added from extending - # the Microsoft 1ES template required for corporate security compliance. Updated again to 120 to mitigate a - # series of build breaks due to timeouts. - timeoutInMinutes: 120 - variables: - - group: ado-feeds - - group: storage-vars - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - name: targetBranchName - value: $(System.PullRequest.TargetBranch) - steps: - # Setup - - checkout: self - clean: true - lfs: '${{ parameters.checkoutSubmodules }}' - submodules: '${{ parameters.checkoutSubmodules }}' - - - task: Bash@3 - displayName: Parameters - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep - # even in an error case - - # Show all task group conditions - - echo " - Pipeline Variables: - - Override Parameters: - packageTypesOverride=${{ parameters.packageTypesOverride }} - publishOverride=${{ parameters.publishOverride }} - releaseBuildOverride=${{ parameters.releaseBuildOverride }} - - Tasks Parameters: - Build=${{ parameters.taskBuild }} - BuildDir=${{ parameters.buildDirectory }} - BuildDoc=${{ parameters.taskBuildDocs }} - Lint=${{ parameters.taskLint }} - LintName: ${{ parameters.taskLintName }} - PublishDocs=${{ parameters.publishDocs }} - Test=${{ convertToJson(parameters.taskTest) }} - TestCoverage=${{ parameters.testCoverage }} - TestResultDirs=${{ convertToJson(parameters.testResultDirs) }} - - Variables: - pathToTelemetryGenerator=$(pathToTelemetryGenerator) - BuildReason=${{ variables['Build.Reason'] }} - - Publish Parameters: - interdependencyRange='${{ parameters.interdependencyRange }}' - packageTypesOverride='${{ parameters.packageTypesOverride }}' - publish=${{ parameters.publish }} - - Computed variables: - canRelease=$(canRelease) - release=$(release) - shouldPublish=$(shouldPublish) - " - - # Target Branch variable (PR policy related) - if [[ ${{ variables['Build.Reason'] }} == "PullRequest" ]]; then - echo "TargetBranchName=$(targetBranchName)" - fi - - # Error checking - if [[ "$(release)" == "release" ]]; then - if [[ "$(canRelease)" == "False" ]]; then - echo "##vso[task.logissue type=error]Invalid branch ${{ variables['Build.SourceBranch'] }} for release" - exit -1; - fi - - if [ -f ".releaseGroup" ]; then - grep -e fluid.*[0-9]-[0-9] `find packages -name 'package.json'` - else - grep -e fluid.*[0-9]-[0-9] `find . -name 'package.json'` - fi - - if [[ $? == 0 ]]; then - echo "##vso[task.logissue type=error]Release shouldn't contain prerelease dependencies" - exit -1; - fi - fi - - if [[ "$(release)" == "prerelease" ]]; then - if [[ "${{ parameters.buildNumberInPatch }}" == "true" ]]; then - echo "##vso[task.logissue type=error] Prerelease not allow for builds that put build number as the patch version" - exit -1; - fi - fi - - if [[ "$(release)" != "prerelease" ]]; then - if [[ "${{ parameters.packageTypesOverride }}" == "alpha" || "${{ parameters.packageTypesOverride }}" == "beta" ]]; then - echo "##vso[task.logissue type=error]This release type is not supported. alpha/beta ***prerelease*** is allowed" - exit -1; - fi - fi - - if [[ "$(release)" != "none" ]] && [[ "$(release)" != "" ]]; then - if [[ "${{ parameters.publish }}" != "True" ]]; then - echo "##vso[task.logissue type=error]'$(release)'' is set but package is not published. Either the branch doesn't default to publish or it is skipped." - exit -1; - fi - fi - - - template: /tools/pipelines/templates/include-use-node-version.yml@self - - - template: /tools/pipelines/templates/include-install.yml@self - parameters: - packageManager: '${{ parameters.packageManager }}' - buildDirectory: '${{ parameters.buildDirectory }}' - packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' - - # This check is a workaround. We don't want to set versions for the build-bundle-size-and-code-coverage-artifacts - # pipeline because it is special - it runs a client build but doesn't publish anything. Working around this properly is - # challenging and would create a much bigger change. Since this is the only pipeline that sets these variables to - # true, we use that to determine whether to set versions. - - ${{ if eq(parameters.taskPublishBundleSizeArtifacts, false) }}: - - template: /tools/pipelines/templates/include-set-package-version.yml@self - parameters: - buildDirectory: '${{ parameters.buildDirectory }}' - buildNumberInPatch: ${{ parameters.buildNumberInPatch }} - buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}' - tagName: '${{ parameters.tagName }}' - interdependencyRange: '${{ parameters.interdependencyRange }}' - packageTypesOverride: '${{ parameters.packageTypesOverride }}' - - # Build and Lint - - template: /tools/pipelines/templates/include-build-lint.yml@self - parameters: - taskBuild: '${{ parameters.taskBuild }}' - taskLint: '${{ parameters.taskLint }}' - taskLintName: '${{ parameters.taskLintName }}' - buildDirectory: '${{ parameters.buildDirectory }}' - - - task: Npm@1 - displayName: 'npm run webpack' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run webpack' - - - task: Bash@3 - displayName: Archive Nested Lib/Dist Directories and Their Contents - env: - WORKING_DIRECTORY: '${{ parameters.buildDirectory }}' - inputs: - targetType: filePath - workingDirectory: '${{ parameters.buildDirectory }}' - filePath: $(Build.SourcesDirectory)/scripts/pack-distlib.sh - - - task: CopyFiles@2 - displayName: Copy nested_lib_dist to artifact staging directory - inputs: - sourceFolder: ${{ parameters.buildDirectory }}/nested_lib_dist - targetFolder: $(Build.ArtifactStagingDirectory)/nested_lib_dist - - # Pack - - ${{ if ne(parameters.taskPack, false) }}: - - task: Bash@3 - displayName: npm pack - env: - PACKAGE_MANAGER: '${{ parameters.packageManager }}' - RELEASE_GROUP: '${{ parameters.tagName }}' - STAGING_PATH: $(Build.ArtifactStagingDirectory) - inputs: - targetType: filePath - workingDirectory: '${{ parameters.buildDirectory }}' - filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh - - # At this point we want to publish the artifact with npm-packed packages, and the one with test files, - # but as part of 1ES migration that's now part of templateContext.outputs below. - - # Collect/publish/run bundle analysis - - ${{ if eq(parameters.taskBundleAnalysis, true) }}: - - task: Npm@1 - displayName: 'npm run bundle-analysis:collect' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run bundle-analysis:collect' - - # Copy files so all artifacts we publish end up under the same parent folder. - # The sourceFolder should be wherever the 'npm run bundle-analysis:collect' task places its output. - - task: CopyFiles@2 - displayName: Copy bundleAnalysis files to artifact staging directory - inputs: - sourceFolder: '${{ parameters.buildDirectory }}/artifacts/bundleAnalysis' - targetFolder: $(Build.ArtifactStagingDirectory)/bundleAnalysis - - - # At this point we want to publish the artifact with the bundle size analysis, - # but as part of 1ES migration that's now part of templateContext.outputs below. - - - task: Npm@1 - displayName: run bundle size comparison - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - continueOnError: true - env: - ADO_API_TOKEN: $(System.AccessToken) - DANGER_GITHUB_API_TOKEN: $(githubPublicRepoSecret) - TARGET_BRANCH_NAME: '$(targetBranchName)' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run bundle-analysis:run' - - - ${{ if and(or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')), eq(variables['System.TeamProject'], 'internal')) }}: - - task: Bash@3 - displayName: List report.json - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - echo "Build Directory is ${{ parameters.buildDirectory }}"; - BUNDLE_SIZE_TESTS_DIR="$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests"; - echo "Contents of $BUNDLE_SIZE_TESTS_DIR:"; - ls -la $BUNDLE_SIZE_TESTS_DIR; - - - template: /tools/pipelines/templates/include-telemetry-setup.yml@self - parameters: - pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) - - - task: Bash@3 - displayName: Write bundle sizes measurements to Aria/Kusto - inputs: - targetType: inline - workingDirectory: $(pathToTelemetryGenerator) - script: | - set -eu -o pipefail - echo "Writing the following performance tests results to Aria/Kusto" - echo "Report Size:" - ls -la '$(Build.SourcesDirectory)/examples/utils/bundle-size-tests/bundleAnalysis/report.json'; - npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/bundleSizeHandler.js" --dir '$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests'; - - # Docs - - ${{ if ne(parameters.taskBuildDocs, false) }}: - - task: Npm@1 - displayName: 'npm run ci:build:docs' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run ci:build:docs' - - # Copy files so all artifacts we publish end up under the same parent folder. - # The sourceFolder should be wherever the 'npm run ci:build:docs' task places its output. - - task: CopyFiles@2 - displayName: Copy _api-extractor-temp files to artifact staging directory - inputs: - sourceFolder: '${{ parameters.buildDirectory }}/_api-extractor-temp' - targetFolder: $(Build.ArtifactStagingDirectory)/_api-extractor-temp - - # At this point we want to publish the artifact with the _api-extractor-temp folder, - # but as part of 1ES migration that's now part of templateContext.outputs below. - - - ${{ if eq(parameters.packageManager, 'pnpm') }}: - # Reset the pnpm-lock.yaml file since it's been modified by the versioning. But for dependency caching we want - # the cache key (which is based on the contents of the lockfile) to be the unmodified file. So we reset the - # lockfile as the last step so that when the dependency cache is uploaded, the cache key matches what it was - # at the beginning of the CI job. - - task: Bash@3 - displayName: Reset lockfile - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - git checkout HEAD -- pnpm-lock.yaml - - # Prune the pnpm store before it's cached. This removes any deps that are not used by the current build. - - task: Bash@3 - displayName: Prune pnpm store - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - pnpm store prune - - - task: Bash@3 - displayName: Check for extraneous modified files - inputs: - targetType: inline - script: | - # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep - # even in an error case - git status | grep -v -E 'package.json|package-lock.json|packageVersion.ts|lerna.json|.npmrc|build-tools/.npmrc|\(use.*' | grep '^\s' > git_status.log - if [ `cat git_status.log | wc -l` != "0" ]; then - cat git_status.log - echo "##vso[task.logissue type=error]Build should not create extraneous files" - exit -1; - fi - - # This additional build step is used to run step not part of the main build. In build client pipeline, - # this is used to inject telemetry key file and upload built devtools extension. - - ${{ parameters.additionalBuildSteps }} - - templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - output: pipelineArtifact - displayName: Publish Artifact - nested_lib_dist - targetPath: $(Build.ArtifactStagingDirectory)/nested_lib_dist - artifactName: nested_lib_dist - publishLocation: pipeline - - - ${{ if ne(parameters.taskPack, false) }}: - - output: pipelineArtifact - displayName: Publish Artifact - pack - targetPath: $(Build.ArtifactStagingDirectory)/pack - artifactName: pack - publishLocation: pipeline - - - output: pipelineArtifact - displayName: Publish Artifact - Test Files - targetPath: $(Build.ArtifactStagingDirectory)/test-files - artifactName: test-files - publishLocation: pipeline - sbomEnabled: false - - - ${{ if eq(parameters.taskBundleAnalysis, true) }}: - - output: pipelineArtifact - displayName: Publish Artifacts - bundle-analysis - condition: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(${{ parameters.taskPublishBundleSizeArtifacts }}, true) ) - targetPath: $(Build.ArtifactStagingDirectory)/bundleAnalysis - artifactName: bundleAnalysis - sbomEnabled: false - publishLocation: pipeline - - - ${{ if or(eq(parameters.publishDocs, true), eq(parameters.taskBuildDocs, true)) }}: - - output: pipelineArtifact - displayName: Publish Artifact - _api-extractor-temp - targetPath: $(Build.ArtifactStagingDirectory)/_api-extractor-temp - artifactName: _api-extractor-temp - sbomEnabled: false - publishLocation: pipeline - - - job: Coverage_tests - displayName: "Coverage tests" - dependsOn: build - variables: - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - name: targetBranchName - value: $(System.PullRequest.TargetBranch) - # Absolute path to the folder that contains the source code for the telemetry-generator package, which is - # used in a few places in the pipeline to push custom telemetry to Kusto. - - name: absolutePathToTelemetryGenerator - value: $(Build.SourcesDirectory)/tools/telemetry-generator - readonly: true - # We already run CodeQL in the main build job, so we don't need to run it again here. - # Note that we need to disable it in the right way for 1ES pipeline templates, vs manual CodeQL tasks. - - name: ONEES_ENFORCED_CODEQL_ENABLED - value: 'false' - steps: - # Setup - - checkout: self - clean: true - lfs: '${{ parameters.checkoutSubmodules }}' - submodules: '${{ parameters.checkoutSubmodules }}' - - - template: /tools/pipelines/templates/include-use-node-version.yml@self - - - template: /tools/pipelines/templates/include-install.yml@self - parameters: - packageManager: '${{ parameters.packageManager }}' - buildDirectory: '${{ parameters.buildDirectory }}' - packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' - - # We need it in order to run flub where the code coverage comparison logic calls for it - - template: /tools/pipelines/templates/include-install-build-tools.yml@self - parameters: - buildDirectory: ${{ parameters.buildDirectory }} - buildToolsVersionToInstall: repo - pnpmStorePath: $(Pipeline.Workspace)/.pnpm-store - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: nested_lib_dist - targetPath: $(Build.SourcesDirectory) - - - script: | - echo "Extracting nested lib/dist directories and their contents..." - tar --extract --gzip --file $(Build.SourcesDirectory)/nested_lib_dist.tar.gz --directory $(Build.SourcesDirectory) - displayName: Extract Nested Lib/Dist Directories and Their Contents - - # Set variable startTest if everything is good so far and we'll start running tests, - # so that the steps to process/upload test coverage results only run if we got to the point of actually running tests. - - script: | - echo "##vso[task.setvariable variable=startTest]true" - displayName: Start Test - - - ${{ each test in parameters.coverageTests }}: - - template: /tools/pipelines/templates/include-test-task.yml@self - parameters: - taskTestStep: '${{ test.name }}' - buildDirectory: '${{ parameters.buildDirectory }}' - testCoverage: '${{ parameters.testCoverage }}' - - - task: Npm@1 - displayName: 'npm run test:copyresults' - condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run test:copyresults' - - # Test - Upload coverage results - # Some webpacked file using externals introduce file name with quotes in them - # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish - # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) - - task: Bash@3 - displayName: Check for nyc/report directory - condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) - inputs: - targetType: 'inline' - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' - - - task: Bash@3 - displayName: Patch Coverage Results - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - targetType: 'inline' - workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' - script: | - set -eu -o pipefail - sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml - - - task: PublishCodeCoverageResults@2 - displayName: Publish Code Coverage - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml - failIfCoverageEmpty: true - - task: CopyFiles@2 - displayName: Copy code coverage report to artifact staging directory - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' - targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - - task: Bash@3 - displayName: Report Code Coverage Comparison - condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) - continueOnError: false - env: - ADO_API_TOKEN: '$(System.AccessToken)' - GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' - TARGET_BRANCH_NAME: '$(targetBranchName)' - ADO_BUILD_ID: '$(Build.BuildId)' - GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' - GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' - ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 - ADO_CI_BUILD_DEFINITION_ID_PR: 11 - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" - echo "Github PR number: $GITHUB_PR_NUMBER" - echo "ADO Build Number: $ADO_BUILD_ID" - echo "Target Branch Name: $TARGET_BRANCH_NAME" - echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" - echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" - echo "Running code coverage comparison" - flub report codeCoverage --verbose - - # Process test result, include publishing and logging - - template: /tools/pipelines/templates/include-process-test-results.yml@self - parameters: - buildDirectory: '${{ parameters.buildDirectory }}' - testResultDirs: '${{ parameters.testResultDirs }}' - - templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - outputs: - - output: pipelineArtifact - displayName: Publish Artifacts - code-coverage - condition: and( succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - targetPath: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - artifactName: 'codeCoverageAnalysis-$(System.JobAttempt)' - sbomEnabled: false - publishLocation: pipeline - - # Parallel jobs for test tasks - - ${{ each test in parameters.taskTest }}: - - job: Test_${{ test.jobName }} - displayName: "Run Task Test ${{ test.jobName }}" - dependsOn: build - variables: - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - name: targetBranchName - value: $(System.PullRequest.TargetBranch) - # Absolute path to the folder that contains the source code for the telemetry-generator package, which is - # used in a few places in the pipeline to push custom telemetry to Kusto. - - name: absolutePathToTelemetryGenerator - value: $(Build.SourcesDirectory)/tools/telemetry-generator - readonly: true - # We already run CodeQL in the main build job, so we don't need to run it again here. - # Note that we need to disable it in the right way for 1ES pipeline templates, vs manual CodeQL tasks. - - name: ONEES_ENFORCED_CODEQL_ENABLED - value: 'false' - steps: - # Setup - - checkout: self - clean: true - lfs: '${{ parameters.checkoutSubmodules }}' - submodules: '${{ parameters.checkoutSubmodules }}' - - - template: /tools/pipelines/templates/include-use-node-version.yml@self - - - template: /tools/pipelines/templates/include-install.yml@self - parameters: - packageManager: '${{ parameters.packageManager }}' - buildDirectory: '${{ parameters.buildDirectory }}' - packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: nested_lib_dist - targetPath: $(Build.SourcesDirectory) - - - script: | - echo "Extracting nested lib/dist directories and their contents..." - tar --extract --gzip --file $(Build.SourcesDirectory)/nested_lib_dist.tar.gz --directory $(Build.SourcesDirectory) - displayName: Extract Nested Lib/Dist Directories and Their Contents - - # Test - - - ${{ if contains(test.name, 'jest') }}: - - task: Npm@1 - displayName: 'Run build for jest tests' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run ci:build' - - # Set variable startTest if everything is good so far and we'll start running tests, - # so that the steps to process/upload test coverage results only run if we got to the point of actually running tests. - - script: | - echo "##vso[task.setvariable variable=startTest]true" - displayName: Start Test - - - template: /tools/pipelines/templates/include-test-task.yml@self - parameters: - taskTestStep: '${{ test.name }}' - buildDirectory: '${{ parameters.buildDirectory }}' - testCoverage: 'false' - - - task: Npm@1 - displayName: 'npm run test:copyresults' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run test:copyresults' - - - ${{ if contains(test.name, 'tinylicious') }}: - - task: Bash@3 - displayName: Upload tinylicious log - condition: always() - continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) - inputs: - targetType: inline - script: | - set -eu -o pipefail - PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; - if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then - echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; - echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; - else - echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; - fi - - # Process test result, include publishing and logging - - template: /tools/pipelines/templates/include-process-test-results.yml@self - parameters: - buildDirectory: '${{ parameters.buildDirectory }}' - testResultDirs: '${{ parameters.testResultDirs }}' + # - stage: build + # displayName: Build Stage + # dependsOn: [] # this stage doesn't depend on preceding stage + # jobs: + # # Job - Build + # - job: build + # displayName: Build + # ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # timeoutInMinutes: 120 + # ${{ else }}: + # # CI builds run more aggressive compat configurations which can take longer. + # # See "FullCompat" under packages\test\test-version-utils\README.md for more details. + # # At the time of adding this comment, the full compat config is on the smaller side and so + # # CI builds consistently pass with a 60 minutes timeout. However, it will naturally grow + # # over time and it might be necessary to bump it. + # # AB#6680 is also relevant here, which tracks rethinking how and where we run tests (likely with + # # a focus on e2e tests) + # # Note, This was recently updated to 90 minutes to account for the additional build time added from extending + # # the Microsoft 1ES template required for corporate security compliance. Updated again to 120 to mitigate a + # # series of build breaks due to timeouts. + # timeoutInMinutes: 120 + # variables: + # - group: ado-feeds + # - group: storage-vars + # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # - name: targetBranchName + # value: $(System.PullRequest.TargetBranch) + # steps: + # # Setup + # - checkout: self + # clean: true + # lfs: '${{ parameters.checkoutSubmodules }}' + # submodules: '${{ parameters.checkoutSubmodules }}' + + # - task: Bash@3 + # displayName: Parameters + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep + # # even in an error case + + # # Show all task group conditions + + # echo " + # Pipeline Variables: + + # Override Parameters: + # packageTypesOverride=${{ parameters.packageTypesOverride }} + # publishOverride=${{ parameters.publishOverride }} + # releaseBuildOverride=${{ parameters.releaseBuildOverride }} + + # Tasks Parameters: + # Build=${{ parameters.taskBuild }} + # BuildDir=${{ parameters.buildDirectory }} + # BuildDoc=${{ parameters.taskBuildDocs }} + # Lint=${{ parameters.taskLint }} + # LintName: ${{ parameters.taskLintName }} + # PublishDocs=${{ parameters.publishDocs }} + # Test=${{ convertToJson(parameters.taskTest) }} + # TestCoverage=${{ parameters.testCoverage }} + # TestResultDirs=${{ convertToJson(parameters.testResultDirs) }} + + # Variables: + # pathToTelemetryGenerator=$(pathToTelemetryGenerator) + # BuildReason=${{ variables['Build.Reason'] }} + + # Publish Parameters: + # interdependencyRange='${{ parameters.interdependencyRange }}' + # packageTypesOverride='${{ parameters.packageTypesOverride }}' + # publish=${{ parameters.publish }} + + # Computed variables: + # canRelease=$(canRelease) + # release=$(release) + # shouldPublish=$(shouldPublish) + # " + + # # Target Branch variable (PR policy related) + # if [[ ${{ variables['Build.Reason'] }} == "PullRequest" ]]; then + # echo "TargetBranchName=$(targetBranchName)" + # fi + + # # Error checking + # if [[ "$(release)" == "release" ]]; then + # if [[ "$(canRelease)" == "False" ]]; then + # echo "##vso[task.logissue type=error]Invalid branch ${{ variables['Build.SourceBranch'] }} for release" + # exit -1; + # fi + + # if [ -f ".releaseGroup" ]; then + # grep -e fluid.*[0-9]-[0-9] `find packages -name 'package.json'` + # else + # grep -e fluid.*[0-9]-[0-9] `find . -name 'package.json'` + # fi + + # if [[ $? == 0 ]]; then + # echo "##vso[task.logissue type=error]Release shouldn't contain prerelease dependencies" + # exit -1; + # fi + # fi + + # if [[ "$(release)" == "prerelease" ]]; then + # if [[ "${{ parameters.buildNumberInPatch }}" == "true" ]]; then + # echo "##vso[task.logissue type=error] Prerelease not allow for builds that put build number as the patch version" + # exit -1; + # fi + # fi + + # if [[ "$(release)" != "prerelease" ]]; then + # if [[ "${{ parameters.packageTypesOverride }}" == "alpha" || "${{ parameters.packageTypesOverride }}" == "beta" ]]; then + # echo "##vso[task.logissue type=error]This release type is not supported. alpha/beta ***prerelease*** is allowed" + # exit -1; + # fi + # fi + + # if [[ "$(release)" != "none" ]] && [[ "$(release)" != "" ]]; then + # if [[ "${{ parameters.publish }}" != "True" ]]; then + # echo "##vso[task.logissue type=error]'$(release)'' is set but package is not published. Either the branch doesn't default to publish or it is skipped." + # exit -1; + # fi + # fi + + # - template: /tools/pipelines/templates/include-use-node-version.yml@self + + # - template: /tools/pipelines/templates/include-install.yml@self + # parameters: + # packageManager: '${{ parameters.packageManager }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + # packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' + + # # This check is a workaround. We don't want to set versions for the build-bundle-size-and-code-coverage-artifacts + # # pipeline because it is special - it runs a client build but doesn't publish anything. Working around this properly is + # # challenging and would create a much bigger change. Since this is the only pipeline that sets these variables to + # # true, we use that to determine whether to set versions. + # - ${{ if eq(parameters.taskPublishBundleSizeArtifacts, false) }}: + # - template: /tools/pipelines/templates/include-set-package-version.yml@self + # parameters: + # buildDirectory: '${{ parameters.buildDirectory }}' + # buildNumberInPatch: ${{ parameters.buildNumberInPatch }} + # buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}' + # tagName: '${{ parameters.tagName }}' + # interdependencyRange: '${{ parameters.interdependencyRange }}' + # packageTypesOverride: '${{ parameters.packageTypesOverride }}' + + # # Build and Lint + # - template: /tools/pipelines/templates/include-build-lint.yml@self + # parameters: + # taskBuild: '${{ parameters.taskBuild }}' + # taskLint: '${{ parameters.taskLint }}' + # taskLintName: '${{ parameters.taskLintName }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + + # - task: Npm@1 + # displayName: 'npm run webpack' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run webpack' + + # - task: Bash@3 + # displayName: Archive Nested Lib/Dist Directories and Their Contents + # env: + # WORKING_DIRECTORY: '${{ parameters.buildDirectory }}' + # inputs: + # targetType: filePath + # workingDirectory: '${{ parameters.buildDirectory }}' + # filePath: $(Build.SourcesDirectory)/scripts/pack-distlib.sh + + # - task: CopyFiles@2 + # displayName: Copy nested_lib_dist to artifact staging directory + # inputs: + # sourceFolder: ${{ parameters.buildDirectory }}/nested_lib_dist + # targetFolder: $(Build.ArtifactStagingDirectory)/nested_lib_dist + + # # Pack + # - ${{ if ne(parameters.taskPack, false) }}: + # - task: Bash@3 + # displayName: npm pack + # env: + # PACKAGE_MANAGER: '${{ parameters.packageManager }}' + # RELEASE_GROUP: '${{ parameters.tagName }}' + # STAGING_PATH: $(Build.ArtifactStagingDirectory) + # inputs: + # targetType: filePath + # workingDirectory: '${{ parameters.buildDirectory }}' + # filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh + + # # At this point we want to publish the artifact with npm-packed packages, and the one with test files, + # # but as part of 1ES migration that's now part of templateContext.outputs below. + + # # Collect/publish/run bundle analysis + # - ${{ if eq(parameters.taskBundleAnalysis, true) }}: + # - task: Npm@1 + # displayName: 'npm run bundle-analysis:collect' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run bundle-analysis:collect' + + # # Copy files so all artifacts we publish end up under the same parent folder. + # # The sourceFolder should be wherever the 'npm run bundle-analysis:collect' task places its output. + # - task: CopyFiles@2 + # displayName: Copy bundleAnalysis files to artifact staging directory + # inputs: + # sourceFolder: '${{ parameters.buildDirectory }}/artifacts/bundleAnalysis' + # targetFolder: $(Build.ArtifactStagingDirectory)/bundleAnalysis + + + # # At this point we want to publish the artifact with the bundle size analysis, + # # but as part of 1ES migration that's now part of templateContext.outputs below. + + # - task: Npm@1 + # displayName: run bundle size comparison + # condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + # continueOnError: true + # env: + # ADO_API_TOKEN: $(System.AccessToken) + # DANGER_GITHUB_API_TOKEN: $(githubPublicRepoSecret) + # TARGET_BRANCH_NAME: '$(targetBranchName)' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run bundle-analysis:run' + + # - ${{ if and(or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')), eq(variables['System.TeamProject'], 'internal')) }}: + # - task: Bash@3 + # displayName: List report.json + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # echo "Build Directory is ${{ parameters.buildDirectory }}"; + # BUNDLE_SIZE_TESTS_DIR="$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests"; + # echo "Contents of $BUNDLE_SIZE_TESTS_DIR:"; + # ls -la $BUNDLE_SIZE_TESTS_DIR; + + # - template: /tools/pipelines/templates/include-telemetry-setup.yml@self + # parameters: + # pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) + + # - task: Bash@3 + # displayName: Write bundle sizes measurements to Aria/Kusto + # inputs: + # targetType: inline + # workingDirectory: $(pathToTelemetryGenerator) + # script: | + # set -eu -o pipefail + # echo "Writing the following performance tests results to Aria/Kusto" + # echo "Report Size:" + # ls -la '$(Build.SourcesDirectory)/examples/utils/bundle-size-tests/bundleAnalysis/report.json'; + # npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/bundleSizeHandler.js" --dir '$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests'; + + # # Docs + # - ${{ if ne(parameters.taskBuildDocs, false) }}: + # - task: Npm@1 + # displayName: 'npm run ci:build:docs' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run ci:build:docs' + + # # Copy files so all artifacts we publish end up under the same parent folder. + # # The sourceFolder should be wherever the 'npm run ci:build:docs' task places its output. + # - task: CopyFiles@2 + # displayName: Copy _api-extractor-temp files to artifact staging directory + # inputs: + # sourceFolder: '${{ parameters.buildDirectory }}/_api-extractor-temp' + # targetFolder: $(Build.ArtifactStagingDirectory)/_api-extractor-temp + + # # At this point we want to publish the artifact with the _api-extractor-temp folder, + # # but as part of 1ES migration that's now part of templateContext.outputs below. + + # - ${{ if eq(parameters.packageManager, 'pnpm') }}: + # # Reset the pnpm-lock.yaml file since it's been modified by the versioning. But for dependency caching we want + # # the cache key (which is based on the contents of the lockfile) to be the unmodified file. So we reset the + # # lockfile as the last step so that when the dependency cache is uploaded, the cache key matches what it was + # # at the beginning of the CI job. + # - task: Bash@3 + # displayName: Reset lockfile + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # git checkout HEAD -- pnpm-lock.yaml + + # # Prune the pnpm store before it's cached. This removes any deps that are not used by the current build. + # - task: Bash@3 + # displayName: Prune pnpm store + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # pnpm store prune + + # - task: Bash@3 + # displayName: Check for extraneous modified files + # inputs: + # targetType: inline + # script: | + # # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep + # # even in an error case + # git status | grep -v -E 'package.json|package-lock.json|packageVersion.ts|lerna.json|.npmrc|build-tools/.npmrc|\(use.*' | grep '^\s' > git_status.log + # if [ `cat git_status.log | wc -l` != "0" ]; then + # cat git_status.log + # echo "##vso[task.logissue type=error]Build should not create extraneous files" + # exit -1; + # fi + + # # This additional build step is used to run step not part of the main build. In build client pipeline, + # # this is used to inject telemetry key file and upload built devtools extension. + # - ${{ parameters.additionalBuildSteps }} + + # templateContext: + # outputParentDirectory: $(Build.ArtifactStagingDirectory) + # outputs: + # - output: pipelineArtifact + # displayName: Publish Artifact - nested_lib_dist + # targetPath: $(Build.ArtifactStagingDirectory)/nested_lib_dist + # artifactName: nested_lib_dist + # publishLocation: pipeline + + # - ${{ if ne(parameters.taskPack, false) }}: + # - output: pipelineArtifact + # displayName: Publish Artifact - pack + # targetPath: $(Build.ArtifactStagingDirectory)/pack + # artifactName: pack + # publishLocation: pipeline + + # - output: pipelineArtifact + # displayName: Publish Artifact - Test Files + # targetPath: $(Build.ArtifactStagingDirectory)/test-files + # artifactName: test-files + # publishLocation: pipeline + # sbomEnabled: false + + # - ${{ if eq(parameters.taskBundleAnalysis, true) }}: + # - output: pipelineArtifact + # displayName: Publish Artifacts - bundle-analysis + # condition: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(${{ parameters.taskPublishBundleSizeArtifacts }}, true) ) + # targetPath: $(Build.ArtifactStagingDirectory)/bundleAnalysis + # artifactName: bundleAnalysis + # sbomEnabled: false + # publishLocation: pipeline + + # - ${{ if or(eq(parameters.publishDocs, true), eq(parameters.taskBuildDocs, true)) }}: + # - output: pipelineArtifact + # displayName: Publish Artifact - _api-extractor-temp + # targetPath: $(Build.ArtifactStagingDirectory)/_api-extractor-temp + # artifactName: _api-extractor-temp + # sbomEnabled: false + # publishLocation: pipeline + + # - job: Coverage_tests + # displayName: "Coverage tests" + # dependsOn: build + # variables: + # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # - name: targetBranchName + # value: $(System.PullRequest.TargetBranch) + # # Absolute path to the folder that contains the source code for the telemetry-generator package, which is + # # used in a few places in the pipeline to push custom telemetry to Kusto. + # - name: absolutePathToTelemetryGenerator + # value: $(Build.SourcesDirectory)/tools/telemetry-generator + # readonly: true + # # We already run CodeQL in the main build job, so we don't need to run it again here. + # # Note that we need to disable it in the right way for 1ES pipeline templates, vs manual CodeQL tasks. + # - name: ONEES_ENFORCED_CODEQL_ENABLED + # value: 'false' + # steps: + # # Setup + # - checkout: self + # clean: true + # lfs: '${{ parameters.checkoutSubmodules }}' + # submodules: '${{ parameters.checkoutSubmodules }}' + + # - template: /tools/pipelines/templates/include-use-node-version.yml@self + + # - template: /tools/pipelines/templates/include-install.yml@self + # parameters: + # packageManager: '${{ parameters.packageManager }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + # packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' + + # # We need it in order to run flub where the code coverage comparison logic calls for it + # - template: /tools/pipelines/templates/include-install-build-tools.yml@self + # parameters: + # buildDirectory: ${{ parameters.buildDirectory }} + # buildToolsVersionToInstall: repo + # pnpmStorePath: $(Pipeline.Workspace)/.pnpm-store + + # - task: DownloadPipelineArtifact@2 + # inputs: + # artifact: nested_lib_dist + # targetPath: $(Build.SourcesDirectory) + + # - script: | + # echo "Extracting nested lib/dist directories and their contents..." + # tar --extract --gzip --file $(Build.SourcesDirectory)/nested_lib_dist.tar.gz --directory $(Build.SourcesDirectory) + # displayName: Extract Nested Lib/Dist Directories and Their Contents + + # # Set variable startTest if everything is good so far and we'll start running tests, + # # so that the steps to process/upload test coverage results only run if we got to the point of actually running tests. + # - script: | + # echo "##vso[task.setvariable variable=startTest]true" + # displayName: Start Test + + # - ${{ each test in parameters.coverageTests }}: + # - template: /tools/pipelines/templates/include-test-task.yml@self + # parameters: + # taskTestStep: '${{ test.name }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + # testCoverage: '${{ parameters.testCoverage }}' + + # - task: Npm@1 + # displayName: 'npm run test:copyresults' + # condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run test:copyresults' + + # # Test - Upload coverage results + # # Some webpacked file using externals introduce file name with quotes in them + # # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish + # # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) + # - task: Bash@3 + # displayName: Check for nyc/report directory + # condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) + # inputs: + # targetType: 'inline' + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' + + # - task: Bash@3 + # displayName: Patch Coverage Results + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # targetType: 'inline' + # workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' + # script: | + # set -eu -o pipefail + # sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml + + # - task: PublishCodeCoverageResults@2 + # displayName: Publish Code Coverage + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml + # failIfCoverageEmpty: true + # - task: CopyFiles@2 + # displayName: Copy code coverage report to artifact staging directory + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' + # targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + # - task: Bash@3 + # displayName: Report Code Coverage Comparison + # condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) + # continueOnError: false + # env: + # ADO_API_TOKEN: '$(System.AccessToken)' + # GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' + # TARGET_BRANCH_NAME: '$(targetBranchName)' + # ADO_BUILD_ID: '$(Build.BuildId)' + # GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' + # GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' + # ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 + # ADO_CI_BUILD_DEFINITION_ID_PR: 11 + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" + # echo "Github PR number: $GITHUB_PR_NUMBER" + # echo "ADO Build Number: $ADO_BUILD_ID" + # echo "Target Branch Name: $TARGET_BRANCH_NAME" + # echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" + # echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" + # echo "Running code coverage comparison" + # flub report codeCoverage --verbose + + # # Process test result, include publishing and logging + # - template: /tools/pipelines/templates/include-process-test-results.yml@self + # parameters: + # buildDirectory: '${{ parameters.buildDirectory }}' + # testResultDirs: '${{ parameters.testResultDirs }}' + + # templateContext: + # outputParentDirectory: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + # outputs: + # - output: pipelineArtifact + # displayName: Publish Artifacts - code-coverage + # condition: and( succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # targetPath: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + # artifactName: 'codeCoverageAnalysis-$(System.JobAttempt)' + # sbomEnabled: false + # publishLocation: pipeline + + # # Parallel jobs for test tasks + # - ${{ each test in parameters.taskTest }}: + # - job: Test_${{ test.jobName }} + # displayName: "Run Task Test ${{ test.jobName }}" + # dependsOn: build + # variables: + # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # - name: targetBranchName + # value: $(System.PullRequest.TargetBranch) + # # Absolute path to the folder that contains the source code for the telemetry-generator package, which is + # # used in a few places in the pipeline to push custom telemetry to Kusto. + # - name: absolutePathToTelemetryGenerator + # value: $(Build.SourcesDirectory)/tools/telemetry-generator + # readonly: true + # # We already run CodeQL in the main build job, so we don't need to run it again here. + # # Note that we need to disable it in the right way for 1ES pipeline templates, vs manual CodeQL tasks. + # - name: ONEES_ENFORCED_CODEQL_ENABLED + # value: 'false' + # steps: + # # Setup + # - checkout: self + # clean: true + # lfs: '${{ parameters.checkoutSubmodules }}' + # submodules: '${{ parameters.checkoutSubmodules }}' + + # - template: /tools/pipelines/templates/include-use-node-version.yml@self + + # - template: /tools/pipelines/templates/include-install.yml@self + # parameters: + # packageManager: '${{ parameters.packageManager }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + # packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' + + # - task: DownloadPipelineArtifact@2 + # inputs: + # artifact: nested_lib_dist + # targetPath: $(Build.SourcesDirectory) + + # - script: | + # echo "Extracting nested lib/dist directories and their contents..." + # tar --extract --gzip --file $(Build.SourcesDirectory)/nested_lib_dist.tar.gz --directory $(Build.SourcesDirectory) + # displayName: Extract Nested Lib/Dist Directories and Their Contents + + # # Test + + # - ${{ if contains(test.name, 'jest') }}: + # - task: Npm@1 + # displayName: 'Run build for jest tests' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run ci:build' + + # # Set variable startTest if everything is good so far and we'll start running tests, + # # so that the steps to process/upload test coverage results only run if we got to the point of actually running tests. + # - script: | + # echo "##vso[task.setvariable variable=startTest]true" + # displayName: Start Test + + # - template: /tools/pipelines/templates/include-test-task.yml@self + # parameters: + # taskTestStep: '${{ test.name }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + # testCoverage: 'false' + + # - task: Npm@1 + # displayName: 'npm run test:copyresults' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run test:copyresults' + + # - ${{ if contains(test.name, 'tinylicious') }}: + # - task: Bash@3 + # displayName: Upload tinylicious log + # condition: always() + # continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) + # inputs: + # targetType: inline + # script: | + # set -eu -o pipefail + # PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; + # if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then + # echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; + # echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; + # else + # echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; + # fi + + # # Process test result, include publishing and logging + # - template: /tools/pipelines/templates/include-process-test-results.yml@self + # parameters: + # buildDirectory: '${{ parameters.buildDirectory }}' + # testResultDirs: '${{ parameters.testResultDirs }}' # # Publish stage # - ${{ if eq(parameters.publish, true) }}: From 0f89faa418fd47de98bef4362569579f80bb4073 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 16:56:54 +0000 Subject: [PATCH 48/58] comments out another stage for testing --- .../templates/build-npm-client-package.yml | 120 +++++++++--------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 710ee8620a7a..a460cad3af65 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -814,63 +814,63 @@ extends: # buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Capture pipeline stage results - - ${{ if eq(parameters.telemetry, true) }}: - - stage: upload_run_telemetry - displayName: Upload pipeline run telemetry to Kusto - condition: succeededOrFailed() - dependsOn: - - build - # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly. - - publish_npm_internal_test - - publish_npm_internal_build - - publish_npm_public - # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally, - # we create a dependency on it based on the same condition. - # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage. - # At some point it might be preferable to always create the stage, control its execution solely with - # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist. - - ${{ if eq(parameters.isReleaseGroup, true) }}: - - publish_npm_internal_dev - jobs: - - job: upload_run_telemetry - displayName: Upload pipeline run telemetry to Kusto - pool: Small-eastus2 - variables: - - group: ado-feeds - - name: pipelineTelemetryWorkdir - value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput - readonly: true - steps: - - template: /tools/pipelines/templates/include-telemetry-setup.yml@self - parameters: - pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) - - task: Bash@3 - displayName: Get stage timing and result data from ADO - env: - BUILD_ID: $(Build.BuildId) - ADO_API_TOKEN: $(System.AccessToken) - WORK_FOLDER: $(pipelineTelemetryWorkdir) - inputs: - targetType: inline - script: | - set -eu -o pipefail - echo "Creating work folder '$WORK_FOLDER'"; - mkdir -p $WORK_FOLDER; - - echo "Retrieving data from ADO API"; - echo "curl -u \":\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\"" - curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json - - task: Bash@3 - displayName: Submit telemetry for stage timing and result - env: - BUILD_ID: $(Build.BuildId) - PIPELINE: BuildClient - WORK_FOLDER: $(pipelineTelemetryWorkdir) - inputs: - targetType: inline - workingDirectory: $(pathToTelemetryGenerator) - script: | - set -eu -o pipefail - echo "Listing files in '$WORK_FOLDER'" - ls -laR $WORK_FOLDER; - npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/stageTimingRetriever.js" --dir "$WORK_FOLDER"; + # - ${{ if eq(parameters.telemetry, true) }}: + # - stage: upload_run_telemetry + # displayName: Upload pipeline run telemetry to Kusto + # condition: succeededOrFailed() + # dependsOn: + # - build + # # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly. + # - publish_npm_internal_test + # - publish_npm_internal_build + # - publish_npm_public + # # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally, + # # we create a dependency on it based on the same condition. + # # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage. + # # At some point it might be preferable to always create the stage, control its execution solely with + # # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist. + # - ${{ if eq(parameters.isReleaseGroup, true) }}: + # - publish_npm_internal_dev + # jobs: + # - job: upload_run_telemetry + # displayName: Upload pipeline run telemetry to Kusto + # pool: Small-eastus2 + # variables: + # - group: ado-feeds + # - name: pipelineTelemetryWorkdir + # value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput + # readonly: true + # steps: + # - template: /tools/pipelines/templates/include-telemetry-setup.yml@self + # parameters: + # pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) + # - task: Bash@3 + # displayName: Get stage timing and result data from ADO + # env: + # BUILD_ID: $(Build.BuildId) + # ADO_API_TOKEN: $(System.AccessToken) + # WORK_FOLDER: $(pipelineTelemetryWorkdir) + # inputs: + # targetType: inline + # script: | + # set -eu -o pipefail + # echo "Creating work folder '$WORK_FOLDER'"; + # mkdir -p $WORK_FOLDER; + + # echo "Retrieving data from ADO API"; + # echo "curl -u \":\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\"" + # curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json + # - task: Bash@3 + # displayName: Submit telemetry for stage timing and result + # env: + # BUILD_ID: $(Build.BuildId) + # PIPELINE: BuildClient + # WORK_FOLDER: $(pipelineTelemetryWorkdir) + # inputs: + # targetType: inline + # workingDirectory: $(pathToTelemetryGenerator) + # script: | + # set -eu -o pipefail + # echo "Listing files in '$WORK_FOLDER'" + # ls -laR $WORK_FOLDER; + # npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/stageTimingRetriever.js" --dir "$WORK_FOLDER"; From 1ddfbbf147c6b0dcd77d63e33cba2e0fa02f265f Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:00:03 +0000 Subject: [PATCH 49/58] more stage comment outs --- .../pipelines/templates/build-npm-package.yml | 1031 +++++++++-------- 1 file changed, 518 insertions(+), 513 deletions(-) diff --git a/tools/pipelines/templates/build-npm-package.yml b/tools/pipelines/templates/build-npm-package.yml index c308fe9dfc14..deda243eb6f5 100644 --- a/tools/pipelines/templates/build-npm-package.yml +++ b/tools/pipelines/templates/build-npm-package.yml @@ -146,7 +146,12 @@ resources: type: git name: 1ESPipelineTemplates/M365GPT ref: refs/tags/release - + pipelines: + # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. + - pipeline: buildTools-resource + project: internal + source: Build - build-tools + branch: main extends: # The pipeline extends the 1ES pipeline template which will inject different SDL and compliance tasks. # Read more: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/onboarding/overview @@ -187,519 +192,519 @@ extends: dependencyInstallCommand: pnpm install --frozen-lockfile # Install / Build / Test Stage - - stage: build - displayName: Build Stage - dependsOn: [] # this stage doesn't depend on preceding stage - jobs: - # Job - Build - - job: build - displayName: Build - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - timeoutInMinutes: 120 - ${{ else }}: - # CI builds run more aggressive compat configurations which can take longer. - # See "FullCompat" under packages\test\test-version-utils\README.md for more details. - # At the time of adding this comment, the full compat config is on the smaller side and so - # CI builds consistently pass with a 60 minutes timeout. However, it will naturally grow - # over time and it might be necessary to bump it. - # AB#6680 is also relevant here, which tracks rethinking how and where we run tests (likely with - # a focus on e2e tests) - # Note, This was recently updated to 90 minutes to account for the additional build time added from extending - # the Microsoft 1ES template required for corporate security compliance. Updated again to 120 to mitigate a - # series of build breaks due to timeouts. - timeoutInMinutes: 120 - variables: - - group: ado-feeds - - group: storage-vars - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - name: targetBranchName - value: $(System.PullRequest.TargetBranch) - steps: - # Setup - - checkout: self - clean: true - lfs: '${{ parameters.checkoutSubmodules }}' - submodules: '${{ parameters.checkoutSubmodules }}' - - - task: Bash@3 - displayName: Parameters - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep - # even in an error case - - # Show all task group conditions - - echo " - Pipeline Variables: - - Override Parameters: - packageTypesOverride=${{ parameters.packageTypesOverride }} - publishOverride=${{ parameters.publishOverride }} - releaseBuildOverride=${{ parameters.releaseBuildOverride }} - - Tasks Parameters: - Build=${{ parameters.taskBuild }} - BuildDir=${{ parameters.buildDirectory }} - BuildDoc=${{ parameters.taskBuildDocs }} - Lint=${{ parameters.taskLint }} - LintName: ${{ parameters.taskLintName }} - PublishDocs=${{ parameters.publishDocs }} - Test=${{ convertToJson(parameters.taskTest) }} - TestCoverage=${{ parameters.testCoverage }} - TestResultDirs=${{ convertToJson(parameters.testResultDirs) }} - - Variables: - pathToTelemetryGenerator=$(pathToTelemetryGenerator) - BuildReason=${{ variables['Build.Reason'] }} - - Publish Parameters: - interdependencyRange='${{ parameters.interdependencyRange }}' - packageTypesOverride='${{ parameters.packageTypesOverride }}' - publish=${{ parameters.publish }} - - Computed variables: - canRelease=$(canRelease) - release=$(release) - shouldPublish=$(shouldPublish) - " - - # Target Branch variable (PR policy related) - if [[ ${{ variables['Build.Reason'] }} == "PullRequest" ]]; then - echo "TargetBranchName=$(targetBranchName)" - fi - - # Error checking - if [[ "$(release)" == "release" ]]; then - if [[ "$(canRelease)" == "False" ]]; then - echo "##vso[task.logissue type=error]Invalid branch ${{ variables['Build.SourceBranch'] }} for release" - exit -1; - fi - - if [ -f ".releaseGroup" ]; then - grep -e fluid.*[0-9]-[0-9] `find packages -name 'package.json'` - else - grep -e fluid.*[0-9]-[0-9] `find . -name 'package.json'` - fi - - if [[ $? == 0 ]]; then - echo "##vso[task.logissue type=error]Release shouldn't contain prerelease dependencies" - exit -1; - fi - fi - - if [[ "$(release)" == "prerelease" ]]; then - if [[ "${{ parameters.buildNumberInPatch }}" == "true" ]]; then - echo "##vso[task.logissue type=error] Prerelease not allow for builds that put build number as the patch version" - exit -1; - fi - fi - - if [[ "$(release)" != "prerelease" ]]; then - if [[ "${{ parameters.packageTypesOverride }}" == "alpha" || "${{ parameters.packageTypesOverride }}" == "beta" ]]; then - echo "##vso[task.logissue type=error]This release type is not supported. alpha/beta ***prerelease*** is allowed" - exit -1; - fi - fi - - if [[ "$(release)" != "none" ]] && [[ "$(release)" != "" ]]; then - if [[ "${{ parameters.publish }}" != "True" ]]; then - echo "##vso[task.logissue type=error]'$(release)'' is set but package is not published. Either the branch doesn't default to publish or it is skipped." - exit -1; - fi - fi - - - template: /tools/pipelines/templates/include-use-node-version.yml@self - - - template: /tools/pipelines/templates/include-install.yml@self - parameters: - packageManager: '${{ parameters.packageManager }}' - buildDirectory: '${{ parameters.buildDirectory }}' - packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' - - # This check is a workaround. We don't want to set versions for the build-bundle-size-and-code-coverage-artifacts - # pipeline because it is special - it runs a client build but doesn't publish anything. Working around this properly is - # challenging and would create a much bigger change. Since this is the only pipeline that sets these variables to - # true, we use that to determine whether to set versions. - - ${{ if eq(parameters.taskPublishBundleSizeArtifacts, false) }}: - - template: /tools/pipelines/templates/include-set-package-version.yml@self - parameters: - buildDirectory: '${{ parameters.buildDirectory }}' - buildNumberInPatch: ${{ parameters.buildNumberInPatch }} - buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}' - tagName: '${{ parameters.tagName }}' - interdependencyRange: '${{ parameters.interdependencyRange }}' - packageTypesOverride: '${{ parameters.packageTypesOverride }}' - - # Build and Lint - - template: /tools/pipelines/templates/include-build-lint.yml@self - parameters: - taskBuild: '${{ parameters.taskBuild }}' - taskLint: '${{ parameters.taskLint }}' - taskLintName: '${{ parameters.taskLintName }}' - buildDirectory: '${{ parameters.buildDirectory }}' - - # Test - - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: - # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not - - script: | - echo "##vso[task.setvariable variable=startTest]true" - displayName: Start Test - - - ${{ each taskTestStep in parameters.taskTest }}: - - template: /tools/pipelines/templates/include-test-task.yml@self - parameters: - taskTestStep: '${{ taskTestStep }}' - buildDirectory: '${{ parameters.buildDirectory }}' - testCoverage: ${{ - and( - parameters.testCoverage, - ne(variables['Build.Reason'], 'PullRequest') - )}} - - - ${{ if contains(convertToJson(parameters.taskTest), 'tinylicious') }}: - - task: Bash@3 - displayName: Upload tinylicious log - condition: always() - continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) - inputs: - targetType: inline - script: | - set -eu -o pipefail - PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; - if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then - echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; - echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; - else - echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; - fi - - # At this point we want to publish the tinylicious.log artifact, but as part of 1ES migration - # that is now part of templateContext.outputs below. - - # Test - Upload coverage results - # Some webpacked file using externals introduce file name with quotes in them - # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish - # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) - - ${{ if parameters.testCoverage }}: - - task: Bash@3 - displayName: Check for nyc/report directory - condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) - inputs: - targetType: 'inline' - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' - - task: Bash@3 - displayName: Patch Coverage Results - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - targetType: 'inline' - workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' - script: | - set -eu -o pipefail - sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml - - task: PublishCodeCoverageResults@2 - displayName: Publish Code Coverage - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml - failIfCoverageEmpty: true - - task: CopyFiles@2 - displayName: Copy code coverage report to artifact staging directory - condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - inputs: - sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' - targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - - task: Bash@3 - displayName: Report Code Coverage Comparison - condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) - continueOnError: false - env: - ADO_API_TOKEN: '$(System.AccessToken)' - GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' - TARGET_BRANCH_NAME: '$(targetBranchName)' - ADO_BUILD_ID: '$(Build.BuildId)' - GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' - GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' - ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 - ADO_CI_BUILD_DEFINITION_ID_PR: 11 - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" - echo "Github PR number: $GITHUB_PR_NUMBER" - echo "ADO Build Number: $ADO_BUILD_ID" - echo "Target Branch Name: $TARGET_BRANCH_NAME" - echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" - echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" - echo "Running code coverage comparison" - flub report codeCoverage --verbose - - - # Process test result, include publishing and logging - - template: /tools/pipelines/templates/include-process-test-results.yml@self - parameters: - buildDirectory: '${{ parameters.buildDirectory }}' - testResultDirs: '${{ parameters.testResultDirs }}' - - # Pack - - ${{ if ne(parameters.taskPack, false) }}: - - task: Bash@3 - displayName: npm pack - env: - PACKAGE_MANAGER: '${{ parameters.packageManager }}' - RELEASE_GROUP: '${{ parameters.tagName }}' - STAGING_PATH: $(Build.ArtifactStagingDirectory) - inputs: - targetType: filePath - workingDirectory: '${{ parameters.buildDirectory }}' - filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh - - # At this point we want to publish the artifact with npm-packed packages, and the one with test files, - # but as part of 1ES migration that's now part of templateContext.outputs below. - - # Collect/publish/run bundle analysis - - ${{ if eq(parameters.taskBundleAnalysis, true) }}: - - task: Npm@1 - displayName: 'npm run bundle-analysis:collect' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run bundle-analysis:collect' - - # Copy files so all artifacts we publish end up under the same parent folder. - # The sourceFolder should be wherever the 'npm run bundle-analysis:collect' task places its output. - - task: CopyFiles@2 - displayName: Copy bundleAnalysis files to artifact staging directory - inputs: - sourceFolder: '${{ parameters.buildDirectory }}/artifacts/bundleAnalysis' - targetFolder: $(Build.ArtifactStagingDirectory)/bundleAnalysis - - - # At this point we want to publish the artifact with the bundle size analysis, - # but as part of 1ES migration that's now part of templateContext.outputs below. - - - task: Npm@1 - displayName: run bundle size comparison - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - continueOnError: true - env: - ADO_API_TOKEN: $(System.AccessToken) - DANGER_GITHUB_API_TOKEN: $(githubPublicRepoSecret) - TARGET_BRANCH_NAME: '$(targetBranchName)' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run bundle-analysis:run' - - - ${{ if and(or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')), eq(variables['System.TeamProject'], 'internal')) }}: - - task: Bash@3 - displayName: List report.json - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - echo "Build Directory is ${{ parameters.buildDirectory }}"; - BUNDLE_SIZE_TESTS_DIR="$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests"; - echo "Contents of $BUNDLE_SIZE_TESTS_DIR:"; - ls -la $BUNDLE_SIZE_TESTS_DIR; - - - template: /tools/pipelines/templates/include-telemetry-setup.yml@self - parameters: - pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) - - - task: Bash@3 - displayName: Write bundle sizes measurements to Aria/Kusto - inputs: - targetType: inline - workingDirectory: $(pathToTelemetryGenerator) - script: | - set -eu -o pipefail - echo "Writing the following performance tests results to Aria/Kusto" - echo "Report Size:" - ls -la '$(Build.SourcesDirectory)/examples/utils/bundle-size-tests/bundleAnalysis/report.json'; - npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/bundleSizeHandler.js" --dir '$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests'; - - # Docs - - ${{ if ne(parameters.taskBuildDocs, false) }}: - - task: Npm@1 - displayName: 'npm run ci:build:docs' - inputs: - command: custom - workingDir: '${{ parameters.buildDirectory }}' - customCommand: 'run ci:build:docs' - - # Copy files so all artifacts we publish end up under the same parent folder. - # The sourceFolder should be wherever the 'npm run ci:build:docs' task places its output. - - task: CopyFiles@2 - displayName: Copy _api-extractor-temp files to artifact staging directory - inputs: - sourceFolder: '${{ parameters.buildDirectory }}/_api-extractor-temp' - targetFolder: $(Build.ArtifactStagingDirectory)/_api-extractor-temp - - # At this point we want to publish the artifact with the _api-extractor-temp folder, - # but as part of 1ES migration that's now part of templateContext.outputs below. - - - ${{ if eq(parameters.packageManager, 'pnpm') }}: - # Reset the pnpm-lock.yaml file since it's been modified by the versioning. But for dependency caching we want - # the cache key (which is based on the contents of the lockfile) to be the unmodified file. So we reset the - # lockfile as the last step so that when the dependency cache is uploaded, the cache key matches what it was - # at the beginning of the CI job. - - task: Bash@3 - displayName: Reset lockfile - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - git checkout HEAD -- pnpm-lock.yaml - - # Prune the pnpm store before it's cached. This removes any deps that are not used by the current build. - - task: Bash@3 - displayName: Prune pnpm store - inputs: - targetType: inline - workingDirectory: '${{ parameters.buildDirectory }}' - script: | - set -eu -o pipefail - pnpm store prune - - - task: Bash@3 - displayName: Check for extraneous modified files - inputs: - targetType: inline - script: | - # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep - # even in an error case - git status | grep -v -E 'package.json|package-lock.json|packageVersion.ts|lerna.json|.npmrc|build-tools/.npmrc|\(use.*' | grep '^\s' > git_status.log - if [ `cat git_status.log | wc -l` != "0" ]; then - cat git_status.log - echo "##vso[task.logissue type=error]Build should not create extraneous files" - exit -1; - fi - - # This additional build step is used to run step not part of the main build. In build client pipeline, - # this is used to inject telemetry key file and upload built devtools extension. - - ${{ parameters.additionalBuildSteps }} - - templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ if ne(parameters.taskPack, false) }}: - - output: pipelineArtifact - displayName: Publish Artifact - pack - targetPath: $(Build.ArtifactStagingDirectory)/pack - artifactName: pack - publishLocation: pipeline - - - output: pipelineArtifact - displayName: Publish Artifact - Test Files - targetPath: $(Build.ArtifactStagingDirectory)/test-files - artifactName: test-files - publishLocation: pipeline - sbomEnabled: false - - - ${{ if eq(parameters.taskBundleAnalysis, true) }}: - - output: pipelineArtifact - displayName: Publish Artifacts - bundle-analysis - condition: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(${{ parameters.taskPublishBundleSizeArtifacts }}, true) ) - targetPath: $(Build.ArtifactStagingDirectory)/bundleAnalysis - artifactName: bundleAnalysis - sbomEnabled: false - publishLocation: pipeline - - # Publish the Code Coverage artifact in this format as it is easier for devs to find coverage html for each file. - - ${{ if eq(parameters.testCoverage, true) }}: - - output: pipelineArtifact - displayName: Publish Artifacts - code-coverage - condition: and( succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - targetPath: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - artifactName: 'codeCoverageAnalysis-$(System.JobAttempt)' - sbomEnabled: false - publishLocation: pipeline - - - ${{ if or(eq(parameters.publishDocs, true), eq(parameters.taskBuildDocs, true)) }}: - - output: pipelineArtifact - displayName: Publish Artifact - _api-extractor-temp - targetPath: $(Build.ArtifactStagingDirectory)/_api-extractor-temp - artifactName: _api-extractor-temp - sbomEnabled: false - publishLocation: pipeline + # - stage: build + # displayName: Build Stage + # dependsOn: [] # this stage doesn't depend on preceding stage + # jobs: + # # Job - Build + # - job: build + # displayName: Build + # ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # timeoutInMinutes: 120 + # ${{ else }}: + # # CI builds run more aggressive compat configurations which can take longer. + # # See "FullCompat" under packages\test\test-version-utils\README.md for more details. + # # At the time of adding this comment, the full compat config is on the smaller side and so + # # CI builds consistently pass with a 60 minutes timeout. However, it will naturally grow + # # over time and it might be necessary to bump it. + # # AB#6680 is also relevant here, which tracks rethinking how and where we run tests (likely with + # # a focus on e2e tests) + # # Note, This was recently updated to 90 minutes to account for the additional build time added from extending + # # the Microsoft 1ES template required for corporate security compliance. Updated again to 120 to mitigate a + # # series of build breaks due to timeouts. + # timeoutInMinutes: 120 + # variables: + # - group: ado-feeds + # - group: storage-vars + # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # - name: targetBranchName + # value: $(System.PullRequest.TargetBranch) + # steps: + # # Setup + # - checkout: self + # clean: true + # lfs: '${{ parameters.checkoutSubmodules }}' + # submodules: '${{ parameters.checkoutSubmodules }}' + + # - task: Bash@3 + # displayName: Parameters + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep + # # even in an error case + + # # Show all task group conditions + + # echo " + # Pipeline Variables: + + # Override Parameters: + # packageTypesOverride=${{ parameters.packageTypesOverride }} + # publishOverride=${{ parameters.publishOverride }} + # releaseBuildOverride=${{ parameters.releaseBuildOverride }} + + # Tasks Parameters: + # Build=${{ parameters.taskBuild }} + # BuildDir=${{ parameters.buildDirectory }} + # BuildDoc=${{ parameters.taskBuildDocs }} + # Lint=${{ parameters.taskLint }} + # LintName: ${{ parameters.taskLintName }} + # PublishDocs=${{ parameters.publishDocs }} + # Test=${{ convertToJson(parameters.taskTest) }} + # TestCoverage=${{ parameters.testCoverage }} + # TestResultDirs=${{ convertToJson(parameters.testResultDirs) }} + + # Variables: + # pathToTelemetryGenerator=$(pathToTelemetryGenerator) + # BuildReason=${{ variables['Build.Reason'] }} + + # Publish Parameters: + # interdependencyRange='${{ parameters.interdependencyRange }}' + # packageTypesOverride='${{ parameters.packageTypesOverride }}' + # publish=${{ parameters.publish }} + + # Computed variables: + # canRelease=$(canRelease) + # release=$(release) + # shouldPublish=$(shouldPublish) + # " + + # # Target Branch variable (PR policy related) + # if [[ ${{ variables['Build.Reason'] }} == "PullRequest" ]]; then + # echo "TargetBranchName=$(targetBranchName)" + # fi + + # # Error checking + # if [[ "$(release)" == "release" ]]; then + # if [[ "$(canRelease)" == "False" ]]; then + # echo "##vso[task.logissue type=error]Invalid branch ${{ variables['Build.SourceBranch'] }} for release" + # exit -1; + # fi + + # if [ -f ".releaseGroup" ]; then + # grep -e fluid.*[0-9]-[0-9] `find packages -name 'package.json'` + # else + # grep -e fluid.*[0-9]-[0-9] `find . -name 'package.json'` + # fi + + # if [[ $? == 0 ]]; then + # echo "##vso[task.logissue type=error]Release shouldn't contain prerelease dependencies" + # exit -1; + # fi + # fi + + # if [[ "$(release)" == "prerelease" ]]; then + # if [[ "${{ parameters.buildNumberInPatch }}" == "true" ]]; then + # echo "##vso[task.logissue type=error] Prerelease not allow for builds that put build number as the patch version" + # exit -1; + # fi + # fi + + # if [[ "$(release)" != "prerelease" ]]; then + # if [[ "${{ parameters.packageTypesOverride }}" == "alpha" || "${{ parameters.packageTypesOverride }}" == "beta" ]]; then + # echo "##vso[task.logissue type=error]This release type is not supported. alpha/beta ***prerelease*** is allowed" + # exit -1; + # fi + # fi + + # if [[ "$(release)" != "none" ]] && [[ "$(release)" != "" ]]; then + # if [[ "${{ parameters.publish }}" != "True" ]]; then + # echo "##vso[task.logissue type=error]'$(release)'' is set but package is not published. Either the branch doesn't default to publish or it is skipped." + # exit -1; + # fi + # fi + + # - template: /tools/pipelines/templates/include-use-node-version.yml@self + + # - template: /tools/pipelines/templates/include-install.yml@self + # parameters: + # packageManager: '${{ parameters.packageManager }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + # packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' + + # # This check is a workaround. We don't want to set versions for the build-bundle-size-and-code-coverage-artifacts + # # pipeline because it is special - it runs a client build but doesn't publish anything. Working around this properly is + # # challenging and would create a much bigger change. Since this is the only pipeline that sets these variables to + # # true, we use that to determine whether to set versions. + # - ${{ if eq(parameters.taskPublishBundleSizeArtifacts, false) }}: + # - template: /tools/pipelines/templates/include-set-package-version.yml@self + # parameters: + # buildDirectory: '${{ parameters.buildDirectory }}' + # buildNumberInPatch: ${{ parameters.buildNumberInPatch }} + # buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}' + # tagName: '${{ parameters.tagName }}' + # interdependencyRange: '${{ parameters.interdependencyRange }}' + # packageTypesOverride: '${{ parameters.packageTypesOverride }}' + + # # Build and Lint + # - template: /tools/pipelines/templates/include-build-lint.yml@self + # parameters: + # taskBuild: '${{ parameters.taskBuild }}' + # taskLint: '${{ parameters.taskLint }}' + # taskLintName: '${{ parameters.taskLintName }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + + # # Test + # - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: + # # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not + # - script: | + # echo "##vso[task.setvariable variable=startTest]true" + # displayName: Start Test + + # - ${{ each taskTestStep in parameters.taskTest }}: + # - template: /tools/pipelines/templates/include-test-task.yml@self + # parameters: + # taskTestStep: '${{ taskTestStep }}' + # buildDirectory: '${{ parameters.buildDirectory }}' + # testCoverage: ${{ + # and( + # parameters.testCoverage, + # ne(variables['Build.Reason'], 'PullRequest') + # )}} + + # - ${{ if contains(convertToJson(parameters.taskTest), 'tinylicious') }}: + # - task: Bash@3 + # displayName: Upload tinylicious log + # condition: always() + # continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) + # inputs: + # targetType: inline + # script: | + # set -eu -o pipefail + # PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; + # if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then + # echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; + # echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; + # else + # echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; + # fi + + # # At this point we want to publish the tinylicious.log artifact, but as part of 1ES migration + # # that is now part of templateContext.outputs below. + + # # Test - Upload coverage results + # # Some webpacked file using externals introduce file name with quotes in them + # # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish + # # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) + # - ${{ if parameters.testCoverage }}: + # - task: Bash@3 + # displayName: Check for nyc/report directory + # condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) + # inputs: + # targetType: 'inline' + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' + # - task: Bash@3 + # displayName: Patch Coverage Results + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # targetType: 'inline' + # workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' + # script: | + # set -eu -o pipefail + # sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml + # - task: PublishCodeCoverageResults@2 + # displayName: Publish Code Coverage + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml + # failIfCoverageEmpty: true + # - task: CopyFiles@2 + # displayName: Copy code coverage report to artifact staging directory + # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # inputs: + # sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' + # targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + # - task: Bash@3 + # displayName: Report Code Coverage Comparison + # condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) + # continueOnError: false + # env: + # ADO_API_TOKEN: '$(System.AccessToken)' + # GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' + # TARGET_BRANCH_NAME: '$(targetBranchName)' + # ADO_BUILD_ID: '$(Build.BuildId)' + # GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' + # GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' + # ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 + # ADO_CI_BUILD_DEFINITION_ID_PR: 11 + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" + # echo "Github PR number: $GITHUB_PR_NUMBER" + # echo "ADO Build Number: $ADO_BUILD_ID" + # echo "Target Branch Name: $TARGET_BRANCH_NAME" + # echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" + # echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" + # echo "Running code coverage comparison" + # flub report codeCoverage --verbose + + + # # Process test result, include publishing and logging + # - template: /tools/pipelines/templates/include-process-test-results.yml@self + # parameters: + # buildDirectory: '${{ parameters.buildDirectory }}' + # testResultDirs: '${{ parameters.testResultDirs }}' + + # # Pack + # - ${{ if ne(parameters.taskPack, false) }}: + # - task: Bash@3 + # displayName: npm pack + # env: + # PACKAGE_MANAGER: '${{ parameters.packageManager }}' + # RELEASE_GROUP: '${{ parameters.tagName }}' + # STAGING_PATH: $(Build.ArtifactStagingDirectory) + # inputs: + # targetType: filePath + # workingDirectory: '${{ parameters.buildDirectory }}' + # filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh + + # # At this point we want to publish the artifact with npm-packed packages, and the one with test files, + # # but as part of 1ES migration that's now part of templateContext.outputs below. + + # # Collect/publish/run bundle analysis + # - ${{ if eq(parameters.taskBundleAnalysis, true) }}: + # - task: Npm@1 + # displayName: 'npm run bundle-analysis:collect' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run bundle-analysis:collect' + + # # Copy files so all artifacts we publish end up under the same parent folder. + # # The sourceFolder should be wherever the 'npm run bundle-analysis:collect' task places its output. + # - task: CopyFiles@2 + # displayName: Copy bundleAnalysis files to artifact staging directory + # inputs: + # sourceFolder: '${{ parameters.buildDirectory }}/artifacts/bundleAnalysis' + # targetFolder: $(Build.ArtifactStagingDirectory)/bundleAnalysis + + + # # At this point we want to publish the artifact with the bundle size analysis, + # # but as part of 1ES migration that's now part of templateContext.outputs below. + + # - task: Npm@1 + # displayName: run bundle size comparison + # condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + # continueOnError: true + # env: + # ADO_API_TOKEN: $(System.AccessToken) + # DANGER_GITHUB_API_TOKEN: $(githubPublicRepoSecret) + # TARGET_BRANCH_NAME: '$(targetBranchName)' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run bundle-analysis:run' + + # - ${{ if and(or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')), eq(variables['System.TeamProject'], 'internal')) }}: + # - task: Bash@3 + # displayName: List report.json + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # echo "Build Directory is ${{ parameters.buildDirectory }}"; + # BUNDLE_SIZE_TESTS_DIR="$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests"; + # echo "Contents of $BUNDLE_SIZE_TESTS_DIR:"; + # ls -la $BUNDLE_SIZE_TESTS_DIR; + + # - template: /tools/pipelines/templates/include-telemetry-setup.yml@self + # parameters: + # pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) + + # - task: Bash@3 + # displayName: Write bundle sizes measurements to Aria/Kusto + # inputs: + # targetType: inline + # workingDirectory: $(pathToTelemetryGenerator) + # script: | + # set -eu -o pipefail + # echo "Writing the following performance tests results to Aria/Kusto" + # echo "Report Size:" + # ls -la '$(Build.SourcesDirectory)/examples/utils/bundle-size-tests/bundleAnalysis/report.json'; + # npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/bundleSizeHandler.js" --dir '$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests'; + + # # Docs + # - ${{ if ne(parameters.taskBuildDocs, false) }}: + # - task: Npm@1 + # displayName: 'npm run ci:build:docs' + # inputs: + # command: custom + # workingDir: '${{ parameters.buildDirectory }}' + # customCommand: 'run ci:build:docs' + + # # Copy files so all artifacts we publish end up under the same parent folder. + # # The sourceFolder should be wherever the 'npm run ci:build:docs' task places its output. + # - task: CopyFiles@2 + # displayName: Copy _api-extractor-temp files to artifact staging directory + # inputs: + # sourceFolder: '${{ parameters.buildDirectory }}/_api-extractor-temp' + # targetFolder: $(Build.ArtifactStagingDirectory)/_api-extractor-temp + + # # At this point we want to publish the artifact with the _api-extractor-temp folder, + # # but as part of 1ES migration that's now part of templateContext.outputs below. + + # - ${{ if eq(parameters.packageManager, 'pnpm') }}: + # # Reset the pnpm-lock.yaml file since it's been modified by the versioning. But for dependency caching we want + # # the cache key (which is based on the contents of the lockfile) to be the unmodified file. So we reset the + # # lockfile as the last step so that when the dependency cache is uploaded, the cache key matches what it was + # # at the beginning of the CI job. + # - task: Bash@3 + # displayName: Reset lockfile + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # git checkout HEAD -- pnpm-lock.yaml + + # # Prune the pnpm store before it's cached. This removes any deps that are not used by the current build. + # - task: Bash@3 + # displayName: Prune pnpm store + # inputs: + # targetType: inline + # workingDirectory: '${{ parameters.buildDirectory }}' + # script: | + # set -eu -o pipefail + # pnpm store prune + + # - task: Bash@3 + # displayName: Check for extraneous modified files + # inputs: + # targetType: inline + # script: | + # # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep + # # even in an error case + # git status | grep -v -E 'package.json|package-lock.json|packageVersion.ts|lerna.json|.npmrc|build-tools/.npmrc|\(use.*' | grep '^\s' > git_status.log + # if [ `cat git_status.log | wc -l` != "0" ]; then + # cat git_status.log + # echo "##vso[task.logissue type=error]Build should not create extraneous files" + # exit -1; + # fi + + # # This additional build step is used to run step not part of the main build. In build client pipeline, + # # this is used to inject telemetry key file and upload built devtools extension. + # - ${{ parameters.additionalBuildSteps }} + + # templateContext: + # outputParentDirectory: $(Build.ArtifactStagingDirectory) + # outputs: + # - ${{ if ne(parameters.taskPack, false) }}: + # - output: pipelineArtifact + # displayName: Publish Artifact - pack + # targetPath: $(Build.ArtifactStagingDirectory)/pack + # artifactName: pack + # publishLocation: pipeline + + # - output: pipelineArtifact + # displayName: Publish Artifact - Test Files + # targetPath: $(Build.ArtifactStagingDirectory)/test-files + # artifactName: test-files + # publishLocation: pipeline + # sbomEnabled: false + + # - ${{ if eq(parameters.taskBundleAnalysis, true) }}: + # - output: pipelineArtifact + # displayName: Publish Artifacts - bundle-analysis + # condition: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(${{ parameters.taskPublishBundleSizeArtifacts }}, true) ) + # targetPath: $(Build.ArtifactStagingDirectory)/bundleAnalysis + # artifactName: bundleAnalysis + # sbomEnabled: false + # publishLocation: pipeline + + # # Publish the Code Coverage artifact in this format as it is easier for devs to find coverage html for each file. + # - ${{ if eq(parameters.testCoverage, true) }}: + # - output: pipelineArtifact + # displayName: Publish Artifacts - code-coverage + # condition: and( succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + # targetPath: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + # artifactName: 'codeCoverageAnalysis-$(System.JobAttempt)' + # sbomEnabled: false + # publishLocation: pipeline + + # - ${{ if or(eq(parameters.publishDocs, true), eq(parameters.taskBuildDocs, true)) }}: + # - output: pipelineArtifact + # displayName: Publish Artifact - _api-extractor-temp + # targetPath: $(Build.ArtifactStagingDirectory)/_api-extractor-temp + # artifactName: _api-extractor-temp + # sbomEnabled: false + # publishLocation: pipeline # Publish stage - - ${{ if eq(parameters.publish, true) }}: - - template: /tools/pipelines/templates/include-publish-npm-package.yml@self - parameters: - tagName: ${{ parameters.tagName }} - isReleaseGroup: ${{ parameters.isReleaseGroup }} - buildDirectory: ${{ parameters.buildDirectory }} - buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} + # - ${{ if eq(parameters.publish, true) }}: + - template: /tools/pipelines/templates/include-publish-npm-package.yml@self + parameters: + tagName: ${{ parameters.tagName }} + isReleaseGroup: ${{ parameters.isReleaseGroup }} + buildDirectory: ${{ parameters.buildDirectory }} + buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Capture pipeline stage results - - ${{ if eq(parameters.telemetry, true) }}: - - stage: upload_run_telemetry - displayName: Upload pipeline run telemetry to Kusto - condition: succeededOrFailed() - dependsOn: - - build - # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly. - - publish_npm_internal_test - - publish_npm_internal_build - - publish_npm_public - # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally, - # we create a dependency on it based on the same condition. - # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage. - # At some point it might be preferable to always create the stage, control its execution solely with - # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist. - - ${{ if eq(parameters.isReleaseGroup, true) }}: - - publish_npm_internal_dev - jobs: - - job: upload_run_telemetry - displayName: Upload pipeline run telemetry to Kusto - pool: Small-eastus2 - variables: - - group: ado-feeds - - name: pipelineTelemetryWorkdir - value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput - readonly: true - steps: - - template: /tools/pipelines/templates/include-telemetry-setup.yml@self - parameters: - pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) - - task: Bash@3 - displayName: Get stage timing and result data from ADO - env: - BUILD_ID: $(Build.BuildId) - ADO_API_TOKEN: $(System.AccessToken) - WORK_FOLDER: $(pipelineTelemetryWorkdir) - inputs: - targetType: inline - script: | - set -eu -o pipefail - echo "Creating work folder '$WORK_FOLDER'"; - mkdir -p $WORK_FOLDER; - - echo "Retrieving data from ADO API"; - echo "curl -u \":\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\"" - curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json - - task: Bash@3 - displayName: Submit telemetry for stage timing and result - env: - BUILD_ID: $(Build.BuildId) - PIPELINE: BuildClient - WORK_FOLDER: $(pipelineTelemetryWorkdir) - inputs: - targetType: inline - workingDirectory: $(pathToTelemetryGenerator) - script: | - set -eu -o pipefail - echo "Listing files in '$WORK_FOLDER'" - ls -laR $WORK_FOLDER; - npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/stageTimingRetriever.js" --dir "$WORK_FOLDER"; + # - ${{ if eq(parameters.telemetry, true) }}: + # - stage: upload_run_telemetry + # displayName: Upload pipeline run telemetry to Kusto + # condition: succeededOrFailed() + # dependsOn: + # - build + # # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly. + # - publish_npm_internal_test + # - publish_npm_internal_build + # - publish_npm_public + # # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally, + # # we create a dependency on it based on the same condition. + # # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage. + # # At some point it might be preferable to always create the stage, control its execution solely with + # # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist. + # - ${{ if eq(parameters.isReleaseGroup, true) }}: + # - publish_npm_internal_dev + # jobs: + # - job: upload_run_telemetry + # displayName: Upload pipeline run telemetry to Kusto + # pool: Small-eastus2 + # variables: + # - group: ado-feeds + # - name: pipelineTelemetryWorkdir + # value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput + # readonly: true + # steps: + # - template: /tools/pipelines/templates/include-telemetry-setup.yml@self + # parameters: + # pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) + # - task: Bash@3 + # displayName: Get stage timing and result data from ADO + # env: + # BUILD_ID: $(Build.BuildId) + # ADO_API_TOKEN: $(System.AccessToken) + # WORK_FOLDER: $(pipelineTelemetryWorkdir) + # inputs: + # targetType: inline + # script: | + # set -eu -o pipefail + # echo "Creating work folder '$WORK_FOLDER'"; + # mkdir -p $WORK_FOLDER; + + # echo "Retrieving data from ADO API"; + # echo "curl -u \":\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\"" + # curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json + # - task: Bash@3 + # displayName: Submit telemetry for stage timing and result + # env: + # BUILD_ID: $(Build.BuildId) + # PIPELINE: BuildClient + # WORK_FOLDER: $(pipelineTelemetryWorkdir) + # inputs: + # targetType: inline + # workingDirectory: $(pathToTelemetryGenerator) + # script: | + # set -eu -o pipefail + # echo "Listing files in '$WORK_FOLDER'" + # ls -laR $WORK_FOLDER; + # npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/stageTimingRetriever.js" --dir "$WORK_FOLDER"; From 006256313e0e0adc148e44e5e771310c0811f28e Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:00:28 +0000 Subject: [PATCH 50/58] comment out pack input to publish job --- .../include-publish-npm-package-deployment.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index e4978c3d5181..c896238f77ae 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -54,11 +54,11 @@ jobs: type: releaseJob isProduction: true # Inputs here are automatically downloaded at the beginning of the job. - inputs: - - input: pipelineArtifact - artifactName: pack - buildType: current - targetPath: $(Pipeline.Workspace)/pack + # inputs: + # - input: pipelineArtifact + # artifactName: pack + # buildType: current + # targetPath: $(Pipeline.Workspace)/pack workspace: clean: all From 6776d10968f927bd5fc35b616fc2af3e5b5bd6b7 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:03:54 +0000 Subject: [PATCH 51/58] Revert "comment out pack input to publish job" This reverts commit 006256313e0e0adc148e44e5e771310c0811f28e. --- .../include-publish-npm-package-deployment.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/pipelines/templates/include-publish-npm-package-deployment.yml b/tools/pipelines/templates/include-publish-npm-package-deployment.yml index c896238f77ae..e4978c3d5181 100644 --- a/tools/pipelines/templates/include-publish-npm-package-deployment.yml +++ b/tools/pipelines/templates/include-publish-npm-package-deployment.yml @@ -54,11 +54,11 @@ jobs: type: releaseJob isProduction: true # Inputs here are automatically downloaded at the beginning of the job. - # inputs: - # - input: pipelineArtifact - # artifactName: pack - # buildType: current - # targetPath: $(Pipeline.Workspace)/pack + inputs: + - input: pipelineArtifact + artifactName: pack + buildType: current + targetPath: $(Pipeline.Workspace)/pack workspace: clean: all From 09985c48c9fad1af47d4901443047388b936246c Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:03:59 +0000 Subject: [PATCH 52/58] Revert "more stage comment outs" This reverts commit 1ddfbbf147c6b0dcd77d63e33cba2e0fa02f265f. --- .../pipelines/templates/build-npm-package.yml | 1031 ++++++++--------- 1 file changed, 513 insertions(+), 518 deletions(-) diff --git a/tools/pipelines/templates/build-npm-package.yml b/tools/pipelines/templates/build-npm-package.yml index deda243eb6f5..c308fe9dfc14 100644 --- a/tools/pipelines/templates/build-npm-package.yml +++ b/tools/pipelines/templates/build-npm-package.yml @@ -146,12 +146,7 @@ resources: type: git name: 1ESPipelineTemplates/M365GPT ref: refs/tags/release - pipelines: - # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. - - pipeline: buildTools-resource - project: internal - source: Build - build-tools - branch: main + extends: # The pipeline extends the 1ES pipeline template which will inject different SDL and compliance tasks. # Read more: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/onboarding/overview @@ -192,519 +187,519 @@ extends: dependencyInstallCommand: pnpm install --frozen-lockfile # Install / Build / Test Stage - # - stage: build - # displayName: Build Stage - # dependsOn: [] # this stage doesn't depend on preceding stage - # jobs: - # # Job - Build - # - job: build - # displayName: Build - # ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # timeoutInMinutes: 120 - # ${{ else }}: - # # CI builds run more aggressive compat configurations which can take longer. - # # See "FullCompat" under packages\test\test-version-utils\README.md for more details. - # # At the time of adding this comment, the full compat config is on the smaller side and so - # # CI builds consistently pass with a 60 minutes timeout. However, it will naturally grow - # # over time and it might be necessary to bump it. - # # AB#6680 is also relevant here, which tracks rethinking how and where we run tests (likely with - # # a focus on e2e tests) - # # Note, This was recently updated to 90 minutes to account for the additional build time added from extending - # # the Microsoft 1ES template required for corporate security compliance. Updated again to 120 to mitigate a - # # series of build breaks due to timeouts. - # timeoutInMinutes: 120 - # variables: - # - group: ado-feeds - # - group: storage-vars - # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # - name: targetBranchName - # value: $(System.PullRequest.TargetBranch) - # steps: - # # Setup - # - checkout: self - # clean: true - # lfs: '${{ parameters.checkoutSubmodules }}' - # submodules: '${{ parameters.checkoutSubmodules }}' - - # - task: Bash@3 - # displayName: Parameters - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep - # # even in an error case - - # # Show all task group conditions - - # echo " - # Pipeline Variables: - - # Override Parameters: - # packageTypesOverride=${{ parameters.packageTypesOverride }} - # publishOverride=${{ parameters.publishOverride }} - # releaseBuildOverride=${{ parameters.releaseBuildOverride }} - - # Tasks Parameters: - # Build=${{ parameters.taskBuild }} - # BuildDir=${{ parameters.buildDirectory }} - # BuildDoc=${{ parameters.taskBuildDocs }} - # Lint=${{ parameters.taskLint }} - # LintName: ${{ parameters.taskLintName }} - # PublishDocs=${{ parameters.publishDocs }} - # Test=${{ convertToJson(parameters.taskTest) }} - # TestCoverage=${{ parameters.testCoverage }} - # TestResultDirs=${{ convertToJson(parameters.testResultDirs) }} - - # Variables: - # pathToTelemetryGenerator=$(pathToTelemetryGenerator) - # BuildReason=${{ variables['Build.Reason'] }} - - # Publish Parameters: - # interdependencyRange='${{ parameters.interdependencyRange }}' - # packageTypesOverride='${{ parameters.packageTypesOverride }}' - # publish=${{ parameters.publish }} - - # Computed variables: - # canRelease=$(canRelease) - # release=$(release) - # shouldPublish=$(shouldPublish) - # " - - # # Target Branch variable (PR policy related) - # if [[ ${{ variables['Build.Reason'] }} == "PullRequest" ]]; then - # echo "TargetBranchName=$(targetBranchName)" - # fi - - # # Error checking - # if [[ "$(release)" == "release" ]]; then - # if [[ "$(canRelease)" == "False" ]]; then - # echo "##vso[task.logissue type=error]Invalid branch ${{ variables['Build.SourceBranch'] }} for release" - # exit -1; - # fi - - # if [ -f ".releaseGroup" ]; then - # grep -e fluid.*[0-9]-[0-9] `find packages -name 'package.json'` - # else - # grep -e fluid.*[0-9]-[0-9] `find . -name 'package.json'` - # fi - - # if [[ $? == 0 ]]; then - # echo "##vso[task.logissue type=error]Release shouldn't contain prerelease dependencies" - # exit -1; - # fi - # fi - - # if [[ "$(release)" == "prerelease" ]]; then - # if [[ "${{ parameters.buildNumberInPatch }}" == "true" ]]; then - # echo "##vso[task.logissue type=error] Prerelease not allow for builds that put build number as the patch version" - # exit -1; - # fi - # fi - - # if [[ "$(release)" != "prerelease" ]]; then - # if [[ "${{ parameters.packageTypesOverride }}" == "alpha" || "${{ parameters.packageTypesOverride }}" == "beta" ]]; then - # echo "##vso[task.logissue type=error]This release type is not supported. alpha/beta ***prerelease*** is allowed" - # exit -1; - # fi - # fi - - # if [[ "$(release)" != "none" ]] && [[ "$(release)" != "" ]]; then - # if [[ "${{ parameters.publish }}" != "True" ]]; then - # echo "##vso[task.logissue type=error]'$(release)'' is set but package is not published. Either the branch doesn't default to publish or it is skipped." - # exit -1; - # fi - # fi - - # - template: /tools/pipelines/templates/include-use-node-version.yml@self - - # - template: /tools/pipelines/templates/include-install.yml@self - # parameters: - # packageManager: '${{ parameters.packageManager }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - # packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' - - # # This check is a workaround. We don't want to set versions for the build-bundle-size-and-code-coverage-artifacts - # # pipeline because it is special - it runs a client build but doesn't publish anything. Working around this properly is - # # challenging and would create a much bigger change. Since this is the only pipeline that sets these variables to - # # true, we use that to determine whether to set versions. - # - ${{ if eq(parameters.taskPublishBundleSizeArtifacts, false) }}: - # - template: /tools/pipelines/templates/include-set-package-version.yml@self - # parameters: - # buildDirectory: '${{ parameters.buildDirectory }}' - # buildNumberInPatch: ${{ parameters.buildNumberInPatch }} - # buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}' - # tagName: '${{ parameters.tagName }}' - # interdependencyRange: '${{ parameters.interdependencyRange }}' - # packageTypesOverride: '${{ parameters.packageTypesOverride }}' - - # # Build and Lint - # - template: /tools/pipelines/templates/include-build-lint.yml@self - # parameters: - # taskBuild: '${{ parameters.taskBuild }}' - # taskLint: '${{ parameters.taskLint }}' - # taskLintName: '${{ parameters.taskLintName }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - - # # Test - # - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: - # # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not - # - script: | - # echo "##vso[task.setvariable variable=startTest]true" - # displayName: Start Test - - # - ${{ each taskTestStep in parameters.taskTest }}: - # - template: /tools/pipelines/templates/include-test-task.yml@self - # parameters: - # taskTestStep: '${{ taskTestStep }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - # testCoverage: ${{ - # and( - # parameters.testCoverage, - # ne(variables['Build.Reason'], 'PullRequest') - # )}} - - # - ${{ if contains(convertToJson(parameters.taskTest), 'tinylicious') }}: - # - task: Bash@3 - # displayName: Upload tinylicious log - # condition: always() - # continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) - # inputs: - # targetType: inline - # script: | - # set -eu -o pipefail - # PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; - # if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then - # echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; - # echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; - # else - # echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; - # fi - - # # At this point we want to publish the tinylicious.log artifact, but as part of 1ES migration - # # that is now part of templateContext.outputs below. - - # # Test - Upload coverage results - # # Some webpacked file using externals introduce file name with quotes in them - # # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish - # # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) - # - ${{ if parameters.testCoverage }}: - # - task: Bash@3 - # displayName: Check for nyc/report directory - # condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) - # inputs: - # targetType: 'inline' - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' - # - task: Bash@3 - # displayName: Patch Coverage Results - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # targetType: 'inline' - # workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' - # script: | - # set -eu -o pipefail - # sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml - # - task: PublishCodeCoverageResults@2 - # displayName: Publish Code Coverage - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml - # failIfCoverageEmpty: true - # - task: CopyFiles@2 - # displayName: Copy code coverage report to artifact staging directory - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' - # targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - # - task: Bash@3 - # displayName: Report Code Coverage Comparison - # condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) - # continueOnError: false - # env: - # ADO_API_TOKEN: '$(System.AccessToken)' - # GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' - # TARGET_BRANCH_NAME: '$(targetBranchName)' - # ADO_BUILD_ID: '$(Build.BuildId)' - # GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' - # GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' - # ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 - # ADO_CI_BUILD_DEFINITION_ID_PR: 11 - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" - # echo "Github PR number: $GITHUB_PR_NUMBER" - # echo "ADO Build Number: $ADO_BUILD_ID" - # echo "Target Branch Name: $TARGET_BRANCH_NAME" - # echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" - # echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" - # echo "Running code coverage comparison" - # flub report codeCoverage --verbose - - - # # Process test result, include publishing and logging - # - template: /tools/pipelines/templates/include-process-test-results.yml@self - # parameters: - # buildDirectory: '${{ parameters.buildDirectory }}' - # testResultDirs: '${{ parameters.testResultDirs }}' - - # # Pack - # - ${{ if ne(parameters.taskPack, false) }}: - # - task: Bash@3 - # displayName: npm pack - # env: - # PACKAGE_MANAGER: '${{ parameters.packageManager }}' - # RELEASE_GROUP: '${{ parameters.tagName }}' - # STAGING_PATH: $(Build.ArtifactStagingDirectory) - # inputs: - # targetType: filePath - # workingDirectory: '${{ parameters.buildDirectory }}' - # filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh - - # # At this point we want to publish the artifact with npm-packed packages, and the one with test files, - # # but as part of 1ES migration that's now part of templateContext.outputs below. - - # # Collect/publish/run bundle analysis - # - ${{ if eq(parameters.taskBundleAnalysis, true) }}: - # - task: Npm@1 - # displayName: 'npm run bundle-analysis:collect' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run bundle-analysis:collect' - - # # Copy files so all artifacts we publish end up under the same parent folder. - # # The sourceFolder should be wherever the 'npm run bundle-analysis:collect' task places its output. - # - task: CopyFiles@2 - # displayName: Copy bundleAnalysis files to artifact staging directory - # inputs: - # sourceFolder: '${{ parameters.buildDirectory }}/artifacts/bundleAnalysis' - # targetFolder: $(Build.ArtifactStagingDirectory)/bundleAnalysis - - - # # At this point we want to publish the artifact with the bundle size analysis, - # # but as part of 1ES migration that's now part of templateContext.outputs below. - - # - task: Npm@1 - # displayName: run bundle size comparison - # condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - # continueOnError: true - # env: - # ADO_API_TOKEN: $(System.AccessToken) - # DANGER_GITHUB_API_TOKEN: $(githubPublicRepoSecret) - # TARGET_BRANCH_NAME: '$(targetBranchName)' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run bundle-analysis:run' - - # - ${{ if and(or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')), eq(variables['System.TeamProject'], 'internal')) }}: - # - task: Bash@3 - # displayName: List report.json - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # echo "Build Directory is ${{ parameters.buildDirectory }}"; - # BUNDLE_SIZE_TESTS_DIR="$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests"; - # echo "Contents of $BUNDLE_SIZE_TESTS_DIR:"; - # ls -la $BUNDLE_SIZE_TESTS_DIR; - - # - template: /tools/pipelines/templates/include-telemetry-setup.yml@self - # parameters: - # pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) - - # - task: Bash@3 - # displayName: Write bundle sizes measurements to Aria/Kusto - # inputs: - # targetType: inline - # workingDirectory: $(pathToTelemetryGenerator) - # script: | - # set -eu -o pipefail - # echo "Writing the following performance tests results to Aria/Kusto" - # echo "Report Size:" - # ls -la '$(Build.SourcesDirectory)/examples/utils/bundle-size-tests/bundleAnalysis/report.json'; - # npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/bundleSizeHandler.js" --dir '$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests'; - - # # Docs - # - ${{ if ne(parameters.taskBuildDocs, false) }}: - # - task: Npm@1 - # displayName: 'npm run ci:build:docs' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run ci:build:docs' - - # # Copy files so all artifacts we publish end up under the same parent folder. - # # The sourceFolder should be wherever the 'npm run ci:build:docs' task places its output. - # - task: CopyFiles@2 - # displayName: Copy _api-extractor-temp files to artifact staging directory - # inputs: - # sourceFolder: '${{ parameters.buildDirectory }}/_api-extractor-temp' - # targetFolder: $(Build.ArtifactStagingDirectory)/_api-extractor-temp - - # # At this point we want to publish the artifact with the _api-extractor-temp folder, - # # but as part of 1ES migration that's now part of templateContext.outputs below. - - # - ${{ if eq(parameters.packageManager, 'pnpm') }}: - # # Reset the pnpm-lock.yaml file since it's been modified by the versioning. But for dependency caching we want - # # the cache key (which is based on the contents of the lockfile) to be the unmodified file. So we reset the - # # lockfile as the last step so that when the dependency cache is uploaded, the cache key matches what it was - # # at the beginning of the CI job. - # - task: Bash@3 - # displayName: Reset lockfile - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # git checkout HEAD -- pnpm-lock.yaml - - # # Prune the pnpm store before it's cached. This removes any deps that are not used by the current build. - # - task: Bash@3 - # displayName: Prune pnpm store - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # pnpm store prune - - # - task: Bash@3 - # displayName: Check for extraneous modified files - # inputs: - # targetType: inline - # script: | - # # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep - # # even in an error case - # git status | grep -v -E 'package.json|package-lock.json|packageVersion.ts|lerna.json|.npmrc|build-tools/.npmrc|\(use.*' | grep '^\s' > git_status.log - # if [ `cat git_status.log | wc -l` != "0" ]; then - # cat git_status.log - # echo "##vso[task.logissue type=error]Build should not create extraneous files" - # exit -1; - # fi - - # # This additional build step is used to run step not part of the main build. In build client pipeline, - # # this is used to inject telemetry key file and upload built devtools extension. - # - ${{ parameters.additionalBuildSteps }} - - # templateContext: - # outputParentDirectory: $(Build.ArtifactStagingDirectory) - # outputs: - # - ${{ if ne(parameters.taskPack, false) }}: - # - output: pipelineArtifact - # displayName: Publish Artifact - pack - # targetPath: $(Build.ArtifactStagingDirectory)/pack - # artifactName: pack - # publishLocation: pipeline - - # - output: pipelineArtifact - # displayName: Publish Artifact - Test Files - # targetPath: $(Build.ArtifactStagingDirectory)/test-files - # artifactName: test-files - # publishLocation: pipeline - # sbomEnabled: false - - # - ${{ if eq(parameters.taskBundleAnalysis, true) }}: - # - output: pipelineArtifact - # displayName: Publish Artifacts - bundle-analysis - # condition: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(${{ parameters.taskPublishBundleSizeArtifacts }}, true) ) - # targetPath: $(Build.ArtifactStagingDirectory)/bundleAnalysis - # artifactName: bundleAnalysis - # sbomEnabled: false - # publishLocation: pipeline - - # # Publish the Code Coverage artifact in this format as it is easier for devs to find coverage html for each file. - # - ${{ if eq(parameters.testCoverage, true) }}: - # - output: pipelineArtifact - # displayName: Publish Artifacts - code-coverage - # condition: and( succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # targetPath: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - # artifactName: 'codeCoverageAnalysis-$(System.JobAttempt)' - # sbomEnabled: false - # publishLocation: pipeline - - # - ${{ if or(eq(parameters.publishDocs, true), eq(parameters.taskBuildDocs, true)) }}: - # - output: pipelineArtifact - # displayName: Publish Artifact - _api-extractor-temp - # targetPath: $(Build.ArtifactStagingDirectory)/_api-extractor-temp - # artifactName: _api-extractor-temp - # sbomEnabled: false - # publishLocation: pipeline + - stage: build + displayName: Build Stage + dependsOn: [] # this stage doesn't depend on preceding stage + jobs: + # Job - Build + - job: build + displayName: Build + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + timeoutInMinutes: 120 + ${{ else }}: + # CI builds run more aggressive compat configurations which can take longer. + # See "FullCompat" under packages\test\test-version-utils\README.md for more details. + # At the time of adding this comment, the full compat config is on the smaller side and so + # CI builds consistently pass with a 60 minutes timeout. However, it will naturally grow + # over time and it might be necessary to bump it. + # AB#6680 is also relevant here, which tracks rethinking how and where we run tests (likely with + # a focus on e2e tests) + # Note, This was recently updated to 90 minutes to account for the additional build time added from extending + # the Microsoft 1ES template required for corporate security compliance. Updated again to 120 to mitigate a + # series of build breaks due to timeouts. + timeoutInMinutes: 120 + variables: + - group: ado-feeds + - group: storage-vars + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - name: targetBranchName + value: $(System.PullRequest.TargetBranch) + steps: + # Setup + - checkout: self + clean: true + lfs: '${{ parameters.checkoutSubmodules }}' + submodules: '${{ parameters.checkoutSubmodules }}' + + - task: Bash@3 + displayName: Parameters + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep + # even in an error case + + # Show all task group conditions + + echo " + Pipeline Variables: + + Override Parameters: + packageTypesOverride=${{ parameters.packageTypesOverride }} + publishOverride=${{ parameters.publishOverride }} + releaseBuildOverride=${{ parameters.releaseBuildOverride }} + + Tasks Parameters: + Build=${{ parameters.taskBuild }} + BuildDir=${{ parameters.buildDirectory }} + BuildDoc=${{ parameters.taskBuildDocs }} + Lint=${{ parameters.taskLint }} + LintName: ${{ parameters.taskLintName }} + PublishDocs=${{ parameters.publishDocs }} + Test=${{ convertToJson(parameters.taskTest) }} + TestCoverage=${{ parameters.testCoverage }} + TestResultDirs=${{ convertToJson(parameters.testResultDirs) }} + + Variables: + pathToTelemetryGenerator=$(pathToTelemetryGenerator) + BuildReason=${{ variables['Build.Reason'] }} + + Publish Parameters: + interdependencyRange='${{ parameters.interdependencyRange }}' + packageTypesOverride='${{ parameters.packageTypesOverride }}' + publish=${{ parameters.publish }} + + Computed variables: + canRelease=$(canRelease) + release=$(release) + shouldPublish=$(shouldPublish) + " + + # Target Branch variable (PR policy related) + if [[ ${{ variables['Build.Reason'] }} == "PullRequest" ]]; then + echo "TargetBranchName=$(targetBranchName)" + fi + + # Error checking + if [[ "$(release)" == "release" ]]; then + if [[ "$(canRelease)" == "False" ]]; then + echo "##vso[task.logissue type=error]Invalid branch ${{ variables['Build.SourceBranch'] }} for release" + exit -1; + fi + + if [ -f ".releaseGroup" ]; then + grep -e fluid.*[0-9]-[0-9] `find packages -name 'package.json'` + else + grep -e fluid.*[0-9]-[0-9] `find . -name 'package.json'` + fi + + if [[ $? == 0 ]]; then + echo "##vso[task.logissue type=error]Release shouldn't contain prerelease dependencies" + exit -1; + fi + fi + + if [[ "$(release)" == "prerelease" ]]; then + if [[ "${{ parameters.buildNumberInPatch }}" == "true" ]]; then + echo "##vso[task.logissue type=error] Prerelease not allow for builds that put build number as the patch version" + exit -1; + fi + fi + + if [[ "$(release)" != "prerelease" ]]; then + if [[ "${{ parameters.packageTypesOverride }}" == "alpha" || "${{ parameters.packageTypesOverride }}" == "beta" ]]; then + echo "##vso[task.logissue type=error]This release type is not supported. alpha/beta ***prerelease*** is allowed" + exit -1; + fi + fi + + if [[ "$(release)" != "none" ]] && [[ "$(release)" != "" ]]; then + if [[ "${{ parameters.publish }}" != "True" ]]; then + echo "##vso[task.logissue type=error]'$(release)'' is set but package is not published. Either the branch doesn't default to publish or it is skipped." + exit -1; + fi + fi + + - template: /tools/pipelines/templates/include-use-node-version.yml@self + + - template: /tools/pipelines/templates/include-install.yml@self + parameters: + packageManager: '${{ parameters.packageManager }}' + buildDirectory: '${{ parameters.buildDirectory }}' + packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' + + # This check is a workaround. We don't want to set versions for the build-bundle-size-and-code-coverage-artifacts + # pipeline because it is special - it runs a client build but doesn't publish anything. Working around this properly is + # challenging and would create a much bigger change. Since this is the only pipeline that sets these variables to + # true, we use that to determine whether to set versions. + - ${{ if eq(parameters.taskPublishBundleSizeArtifacts, false) }}: + - template: /tools/pipelines/templates/include-set-package-version.yml@self + parameters: + buildDirectory: '${{ parameters.buildDirectory }}' + buildNumberInPatch: ${{ parameters.buildNumberInPatch }} + buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}' + tagName: '${{ parameters.tagName }}' + interdependencyRange: '${{ parameters.interdependencyRange }}' + packageTypesOverride: '${{ parameters.packageTypesOverride }}' + + # Build and Lint + - template: /tools/pipelines/templates/include-build-lint.yml@self + parameters: + taskBuild: '${{ parameters.taskBuild }}' + taskLint: '${{ parameters.taskLint }}' + taskLintName: '${{ parameters.taskLintName }}' + buildDirectory: '${{ parameters.buildDirectory }}' + + # Test + - ${{ if ne(convertToJson(parameters.taskTest), '[]') }}: + # Set variable startTest if the build succeed so that we can run all the test tasks whether they are failed or not + - script: | + echo "##vso[task.setvariable variable=startTest]true" + displayName: Start Test + + - ${{ each taskTestStep in parameters.taskTest }}: + - template: /tools/pipelines/templates/include-test-task.yml@self + parameters: + taskTestStep: '${{ taskTestStep }}' + buildDirectory: '${{ parameters.buildDirectory }}' + testCoverage: ${{ + and( + parameters.testCoverage, + ne(variables['Build.Reason'], 'PullRequest') + )}} + + - ${{ if contains(convertToJson(parameters.taskTest), 'tinylicious') }}: + - task: Bash@3 + displayName: Upload tinylicious log + condition: always() + continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) + inputs: + targetType: inline + script: | + set -eu -o pipefail + PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; + if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then + echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; + echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; + else + echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; + fi + + # At this point we want to publish the tinylicious.log artifact, but as part of 1ES migration + # that is now part of templateContext.outputs below. + + # Test - Upload coverage results + # Some webpacked file using externals introduce file name with quotes in them + # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish + # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) + - ${{ if parameters.testCoverage }}: + - task: Bash@3 + displayName: Check for nyc/report directory + condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) + inputs: + targetType: 'inline' + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' + - task: Bash@3 + displayName: Patch Coverage Results + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + targetType: 'inline' + workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' + script: | + set -eu -o pipefail + sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml + - task: PublishCodeCoverageResults@2 + displayName: Publish Code Coverage + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml + failIfCoverageEmpty: true + - task: CopyFiles@2 + displayName: Copy code coverage report to artifact staging directory + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' + targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + - task: Bash@3 + displayName: Report Code Coverage Comparison + condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) + continueOnError: false + env: + ADO_API_TOKEN: '$(System.AccessToken)' + GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' + TARGET_BRANCH_NAME: '$(targetBranchName)' + ADO_BUILD_ID: '$(Build.BuildId)' + GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' + GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' + ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 + ADO_CI_BUILD_DEFINITION_ID_PR: 11 + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" + echo "Github PR number: $GITHUB_PR_NUMBER" + echo "ADO Build Number: $ADO_BUILD_ID" + echo "Target Branch Name: $TARGET_BRANCH_NAME" + echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" + echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" + echo "Running code coverage comparison" + flub report codeCoverage --verbose + + + # Process test result, include publishing and logging + - template: /tools/pipelines/templates/include-process-test-results.yml@self + parameters: + buildDirectory: '${{ parameters.buildDirectory }}' + testResultDirs: '${{ parameters.testResultDirs }}' + + # Pack + - ${{ if ne(parameters.taskPack, false) }}: + - task: Bash@3 + displayName: npm pack + env: + PACKAGE_MANAGER: '${{ parameters.packageManager }}' + RELEASE_GROUP: '${{ parameters.tagName }}' + STAGING_PATH: $(Build.ArtifactStagingDirectory) + inputs: + targetType: filePath + workingDirectory: '${{ parameters.buildDirectory }}' + filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh + + # At this point we want to publish the artifact with npm-packed packages, and the one with test files, + # but as part of 1ES migration that's now part of templateContext.outputs below. + + # Collect/publish/run bundle analysis + - ${{ if eq(parameters.taskBundleAnalysis, true) }}: + - task: Npm@1 + displayName: 'npm run bundle-analysis:collect' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run bundle-analysis:collect' + + # Copy files so all artifacts we publish end up under the same parent folder. + # The sourceFolder should be wherever the 'npm run bundle-analysis:collect' task places its output. + - task: CopyFiles@2 + displayName: Copy bundleAnalysis files to artifact staging directory + inputs: + sourceFolder: '${{ parameters.buildDirectory }}/artifacts/bundleAnalysis' + targetFolder: $(Build.ArtifactStagingDirectory)/bundleAnalysis + + + # At this point we want to publish the artifact with the bundle size analysis, + # but as part of 1ES migration that's now part of templateContext.outputs below. + + - task: Npm@1 + displayName: run bundle size comparison + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + continueOnError: true + env: + ADO_API_TOKEN: $(System.AccessToken) + DANGER_GITHUB_API_TOKEN: $(githubPublicRepoSecret) + TARGET_BRANCH_NAME: '$(targetBranchName)' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run bundle-analysis:run' + + - ${{ if and(or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')), eq(variables['System.TeamProject'], 'internal')) }}: + - task: Bash@3 + displayName: List report.json + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + echo "Build Directory is ${{ parameters.buildDirectory }}"; + BUNDLE_SIZE_TESTS_DIR="$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests"; + echo "Contents of $BUNDLE_SIZE_TESTS_DIR:"; + ls -la $BUNDLE_SIZE_TESTS_DIR; + + - template: /tools/pipelines/templates/include-telemetry-setup.yml@self + parameters: + pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) + + - task: Bash@3 + displayName: Write bundle sizes measurements to Aria/Kusto + inputs: + targetType: inline + workingDirectory: $(pathToTelemetryGenerator) + script: | + set -eu -o pipefail + echo "Writing the following performance tests results to Aria/Kusto" + echo "Report Size:" + ls -la '$(Build.SourcesDirectory)/examples/utils/bundle-size-tests/bundleAnalysis/report.json'; + npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/bundleSizeHandler.js" --dir '$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests'; + + # Docs + - ${{ if ne(parameters.taskBuildDocs, false) }}: + - task: Npm@1 + displayName: 'npm run ci:build:docs' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run ci:build:docs' + + # Copy files so all artifacts we publish end up under the same parent folder. + # The sourceFolder should be wherever the 'npm run ci:build:docs' task places its output. + - task: CopyFiles@2 + displayName: Copy _api-extractor-temp files to artifact staging directory + inputs: + sourceFolder: '${{ parameters.buildDirectory }}/_api-extractor-temp' + targetFolder: $(Build.ArtifactStagingDirectory)/_api-extractor-temp + + # At this point we want to publish the artifact with the _api-extractor-temp folder, + # but as part of 1ES migration that's now part of templateContext.outputs below. + + - ${{ if eq(parameters.packageManager, 'pnpm') }}: + # Reset the pnpm-lock.yaml file since it's been modified by the versioning. But for dependency caching we want + # the cache key (which is based on the contents of the lockfile) to be the unmodified file. So we reset the + # lockfile as the last step so that when the dependency cache is uploaded, the cache key matches what it was + # at the beginning of the CI job. + - task: Bash@3 + displayName: Reset lockfile + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + git checkout HEAD -- pnpm-lock.yaml + + # Prune the pnpm store before it's cached. This removes any deps that are not used by the current build. + - task: Bash@3 + displayName: Prune pnpm store + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + pnpm store prune + + - task: Bash@3 + displayName: Check for extraneous modified files + inputs: + targetType: inline + script: | + # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep + # even in an error case + git status | grep -v -E 'package.json|package-lock.json|packageVersion.ts|lerna.json|.npmrc|build-tools/.npmrc|\(use.*' | grep '^\s' > git_status.log + if [ `cat git_status.log | wc -l` != "0" ]; then + cat git_status.log + echo "##vso[task.logissue type=error]Build should not create extraneous files" + exit -1; + fi + + # This additional build step is used to run step not part of the main build. In build client pipeline, + # this is used to inject telemetry key file and upload built devtools extension. + - ${{ parameters.additionalBuildSteps }} + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ if ne(parameters.taskPack, false) }}: + - output: pipelineArtifact + displayName: Publish Artifact - pack + targetPath: $(Build.ArtifactStagingDirectory)/pack + artifactName: pack + publishLocation: pipeline + + - output: pipelineArtifact + displayName: Publish Artifact - Test Files + targetPath: $(Build.ArtifactStagingDirectory)/test-files + artifactName: test-files + publishLocation: pipeline + sbomEnabled: false + + - ${{ if eq(parameters.taskBundleAnalysis, true) }}: + - output: pipelineArtifact + displayName: Publish Artifacts - bundle-analysis + condition: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(${{ parameters.taskPublishBundleSizeArtifacts }}, true) ) + targetPath: $(Build.ArtifactStagingDirectory)/bundleAnalysis + artifactName: bundleAnalysis + sbomEnabled: false + publishLocation: pipeline + + # Publish the Code Coverage artifact in this format as it is easier for devs to find coverage html for each file. + - ${{ if eq(parameters.testCoverage, true) }}: + - output: pipelineArtifact + displayName: Publish Artifacts - code-coverage + condition: and( succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + targetPath: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + artifactName: 'codeCoverageAnalysis-$(System.JobAttempt)' + sbomEnabled: false + publishLocation: pipeline + + - ${{ if or(eq(parameters.publishDocs, true), eq(parameters.taskBuildDocs, true)) }}: + - output: pipelineArtifact + displayName: Publish Artifact - _api-extractor-temp + targetPath: $(Build.ArtifactStagingDirectory)/_api-extractor-temp + artifactName: _api-extractor-temp + sbomEnabled: false + publishLocation: pipeline # Publish stage - # - ${{ if eq(parameters.publish, true) }}: - - template: /tools/pipelines/templates/include-publish-npm-package.yml@self - parameters: - tagName: ${{ parameters.tagName }} - isReleaseGroup: ${{ parameters.isReleaseGroup }} - buildDirectory: ${{ parameters.buildDirectory }} - buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} + - ${{ if eq(parameters.publish, true) }}: + - template: /tools/pipelines/templates/include-publish-npm-package.yml@self + parameters: + tagName: ${{ parameters.tagName }} + isReleaseGroup: ${{ parameters.isReleaseGroup }} + buildDirectory: ${{ parameters.buildDirectory }} + buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Capture pipeline stage results - # - ${{ if eq(parameters.telemetry, true) }}: - # - stage: upload_run_telemetry - # displayName: Upload pipeline run telemetry to Kusto - # condition: succeededOrFailed() - # dependsOn: - # - build - # # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly. - # - publish_npm_internal_test - # - publish_npm_internal_build - # - publish_npm_public - # # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally, - # # we create a dependency on it based on the same condition. - # # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage. - # # At some point it might be preferable to always create the stage, control its execution solely with - # # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist. - # - ${{ if eq(parameters.isReleaseGroup, true) }}: - # - publish_npm_internal_dev - # jobs: - # - job: upload_run_telemetry - # displayName: Upload pipeline run telemetry to Kusto - # pool: Small-eastus2 - # variables: - # - group: ado-feeds - # - name: pipelineTelemetryWorkdir - # value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput - # readonly: true - # steps: - # - template: /tools/pipelines/templates/include-telemetry-setup.yml@self - # parameters: - # pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) - # - task: Bash@3 - # displayName: Get stage timing and result data from ADO - # env: - # BUILD_ID: $(Build.BuildId) - # ADO_API_TOKEN: $(System.AccessToken) - # WORK_FOLDER: $(pipelineTelemetryWorkdir) - # inputs: - # targetType: inline - # script: | - # set -eu -o pipefail - # echo "Creating work folder '$WORK_FOLDER'"; - # mkdir -p $WORK_FOLDER; - - # echo "Retrieving data from ADO API"; - # echo "curl -u \":\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\"" - # curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json - # - task: Bash@3 - # displayName: Submit telemetry for stage timing and result - # env: - # BUILD_ID: $(Build.BuildId) - # PIPELINE: BuildClient - # WORK_FOLDER: $(pipelineTelemetryWorkdir) - # inputs: - # targetType: inline - # workingDirectory: $(pathToTelemetryGenerator) - # script: | - # set -eu -o pipefail - # echo "Listing files in '$WORK_FOLDER'" - # ls -laR $WORK_FOLDER; - # npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/stageTimingRetriever.js" --dir "$WORK_FOLDER"; + - ${{ if eq(parameters.telemetry, true) }}: + - stage: upload_run_telemetry + displayName: Upload pipeline run telemetry to Kusto + condition: succeededOrFailed() + dependsOn: + - build + # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly. + - publish_npm_internal_test + - publish_npm_internal_build + - publish_npm_public + # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally, + # we create a dependency on it based on the same condition. + # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage. + # At some point it might be preferable to always create the stage, control its execution solely with + # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist. + - ${{ if eq(parameters.isReleaseGroup, true) }}: + - publish_npm_internal_dev + jobs: + - job: upload_run_telemetry + displayName: Upload pipeline run telemetry to Kusto + pool: Small-eastus2 + variables: + - group: ado-feeds + - name: pipelineTelemetryWorkdir + value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput + readonly: true + steps: + - template: /tools/pipelines/templates/include-telemetry-setup.yml@self + parameters: + pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) + - task: Bash@3 + displayName: Get stage timing and result data from ADO + env: + BUILD_ID: $(Build.BuildId) + ADO_API_TOKEN: $(System.AccessToken) + WORK_FOLDER: $(pipelineTelemetryWorkdir) + inputs: + targetType: inline + script: | + set -eu -o pipefail + echo "Creating work folder '$WORK_FOLDER'"; + mkdir -p $WORK_FOLDER; + + echo "Retrieving data from ADO API"; + echo "curl -u \":\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\"" + curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json + - task: Bash@3 + displayName: Submit telemetry for stage timing and result + env: + BUILD_ID: $(Build.BuildId) + PIPELINE: BuildClient + WORK_FOLDER: $(pipelineTelemetryWorkdir) + inputs: + targetType: inline + workingDirectory: $(pathToTelemetryGenerator) + script: | + set -eu -o pipefail + echo "Listing files in '$WORK_FOLDER'" + ls -laR $WORK_FOLDER; + npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/stageTimingRetriever.js" --dir "$WORK_FOLDER"; From c37ec9420f869ffbe279db53ff70853d1e915a7f Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:04:01 +0000 Subject: [PATCH 53/58] Revert "comments out another stage for testing" This reverts commit 0f89faa418fd47de98bef4362569579f80bb4073. --- .../templates/build-npm-client-package.yml | 120 +++++++++--------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index a460cad3af65..710ee8620a7a 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -814,63 +814,63 @@ extends: # buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Capture pipeline stage results - # - ${{ if eq(parameters.telemetry, true) }}: - # - stage: upload_run_telemetry - # displayName: Upload pipeline run telemetry to Kusto - # condition: succeededOrFailed() - # dependsOn: - # - build - # # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly. - # - publish_npm_internal_test - # - publish_npm_internal_build - # - publish_npm_public - # # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally, - # # we create a dependency on it based on the same condition. - # # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage. - # # At some point it might be preferable to always create the stage, control its execution solely with - # # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist. - # - ${{ if eq(parameters.isReleaseGroup, true) }}: - # - publish_npm_internal_dev - # jobs: - # - job: upload_run_telemetry - # displayName: Upload pipeline run telemetry to Kusto - # pool: Small-eastus2 - # variables: - # - group: ado-feeds - # - name: pipelineTelemetryWorkdir - # value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput - # readonly: true - # steps: - # - template: /tools/pipelines/templates/include-telemetry-setup.yml@self - # parameters: - # pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) - # - task: Bash@3 - # displayName: Get stage timing and result data from ADO - # env: - # BUILD_ID: $(Build.BuildId) - # ADO_API_TOKEN: $(System.AccessToken) - # WORK_FOLDER: $(pipelineTelemetryWorkdir) - # inputs: - # targetType: inline - # script: | - # set -eu -o pipefail - # echo "Creating work folder '$WORK_FOLDER'"; - # mkdir -p $WORK_FOLDER; - - # echo "Retrieving data from ADO API"; - # echo "curl -u \":\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\"" - # curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json - # - task: Bash@3 - # displayName: Submit telemetry for stage timing and result - # env: - # BUILD_ID: $(Build.BuildId) - # PIPELINE: BuildClient - # WORK_FOLDER: $(pipelineTelemetryWorkdir) - # inputs: - # targetType: inline - # workingDirectory: $(pathToTelemetryGenerator) - # script: | - # set -eu -o pipefail - # echo "Listing files in '$WORK_FOLDER'" - # ls -laR $WORK_FOLDER; - # npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/stageTimingRetriever.js" --dir "$WORK_FOLDER"; + - ${{ if eq(parameters.telemetry, true) }}: + - stage: upload_run_telemetry + displayName: Upload pipeline run telemetry to Kusto + condition: succeededOrFailed() + dependsOn: + - build + # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly. + - publish_npm_internal_test + - publish_npm_internal_build + - publish_npm_public + # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally, + # we create a dependency on it based on the same condition. + # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage. + # At some point it might be preferable to always create the stage, control its execution solely with + # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist. + - ${{ if eq(parameters.isReleaseGroup, true) }}: + - publish_npm_internal_dev + jobs: + - job: upload_run_telemetry + displayName: Upload pipeline run telemetry to Kusto + pool: Small-eastus2 + variables: + - group: ado-feeds + - name: pipelineTelemetryWorkdir + value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput + readonly: true + steps: + - template: /tools/pipelines/templates/include-telemetry-setup.yml@self + parameters: + pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) + - task: Bash@3 + displayName: Get stage timing and result data from ADO + env: + BUILD_ID: $(Build.BuildId) + ADO_API_TOKEN: $(System.AccessToken) + WORK_FOLDER: $(pipelineTelemetryWorkdir) + inputs: + targetType: inline + script: | + set -eu -o pipefail + echo "Creating work folder '$WORK_FOLDER'"; + mkdir -p $WORK_FOLDER; + + echo "Retrieving data from ADO API"; + echo "curl -u \":\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\"" + curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json + - task: Bash@3 + displayName: Submit telemetry for stage timing and result + env: + BUILD_ID: $(Build.BuildId) + PIPELINE: BuildClient + WORK_FOLDER: $(pipelineTelemetryWorkdir) + inputs: + targetType: inline + workingDirectory: $(pathToTelemetryGenerator) + script: | + set -eu -o pipefail + echo "Listing files in '$WORK_FOLDER'" + ls -laR $WORK_FOLDER; + npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/stageTimingRetriever.js" --dir "$WORK_FOLDER"; From fb88dad1e5e55914fa4177113d0c0ffeed7ed2ef Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:04:03 +0000 Subject: [PATCH 54/58] Revert "comments out build stage" This reverts commit 8fa5a92ab6995133a0cb2accb0d2d4d8fa3fd874. --- .../templates/build-npm-client-package.yml | 1200 ++++++++--------- 1 file changed, 600 insertions(+), 600 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 710ee8620a7a..ff19587e5fef 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -203,606 +203,606 @@ extends: buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Install / Build / Test Stage - # - stage: build - # displayName: Build Stage - # dependsOn: [] # this stage doesn't depend on preceding stage - # jobs: - # # Job - Build - # - job: build - # displayName: Build - # ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # timeoutInMinutes: 120 - # ${{ else }}: - # # CI builds run more aggressive compat configurations which can take longer. - # # See "FullCompat" under packages\test\test-version-utils\README.md for more details. - # # At the time of adding this comment, the full compat config is on the smaller side and so - # # CI builds consistently pass with a 60 minutes timeout. However, it will naturally grow - # # over time and it might be necessary to bump it. - # # AB#6680 is also relevant here, which tracks rethinking how and where we run tests (likely with - # # a focus on e2e tests) - # # Note, This was recently updated to 90 minutes to account for the additional build time added from extending - # # the Microsoft 1ES template required for corporate security compliance. Updated again to 120 to mitigate a - # # series of build breaks due to timeouts. - # timeoutInMinutes: 120 - # variables: - # - group: ado-feeds - # - group: storage-vars - # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # - name: targetBranchName - # value: $(System.PullRequest.TargetBranch) - # steps: - # # Setup - # - checkout: self - # clean: true - # lfs: '${{ parameters.checkoutSubmodules }}' - # submodules: '${{ parameters.checkoutSubmodules }}' - - # - task: Bash@3 - # displayName: Parameters - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep - # # even in an error case - - # # Show all task group conditions - - # echo " - # Pipeline Variables: - - # Override Parameters: - # packageTypesOverride=${{ parameters.packageTypesOverride }} - # publishOverride=${{ parameters.publishOverride }} - # releaseBuildOverride=${{ parameters.releaseBuildOverride }} - - # Tasks Parameters: - # Build=${{ parameters.taskBuild }} - # BuildDir=${{ parameters.buildDirectory }} - # BuildDoc=${{ parameters.taskBuildDocs }} - # Lint=${{ parameters.taskLint }} - # LintName: ${{ parameters.taskLintName }} - # PublishDocs=${{ parameters.publishDocs }} - # Test=${{ convertToJson(parameters.taskTest) }} - # TestCoverage=${{ parameters.testCoverage }} - # TestResultDirs=${{ convertToJson(parameters.testResultDirs) }} - - # Variables: - # pathToTelemetryGenerator=$(pathToTelemetryGenerator) - # BuildReason=${{ variables['Build.Reason'] }} - - # Publish Parameters: - # interdependencyRange='${{ parameters.interdependencyRange }}' - # packageTypesOverride='${{ parameters.packageTypesOverride }}' - # publish=${{ parameters.publish }} - - # Computed variables: - # canRelease=$(canRelease) - # release=$(release) - # shouldPublish=$(shouldPublish) - # " - - # # Target Branch variable (PR policy related) - # if [[ ${{ variables['Build.Reason'] }} == "PullRequest" ]]; then - # echo "TargetBranchName=$(targetBranchName)" - # fi - - # # Error checking - # if [[ "$(release)" == "release" ]]; then - # if [[ "$(canRelease)" == "False" ]]; then - # echo "##vso[task.logissue type=error]Invalid branch ${{ variables['Build.SourceBranch'] }} for release" - # exit -1; - # fi - - # if [ -f ".releaseGroup" ]; then - # grep -e fluid.*[0-9]-[0-9] `find packages -name 'package.json'` - # else - # grep -e fluid.*[0-9]-[0-9] `find . -name 'package.json'` - # fi - - # if [[ $? == 0 ]]; then - # echo "##vso[task.logissue type=error]Release shouldn't contain prerelease dependencies" - # exit -1; - # fi - # fi - - # if [[ "$(release)" == "prerelease" ]]; then - # if [[ "${{ parameters.buildNumberInPatch }}" == "true" ]]; then - # echo "##vso[task.logissue type=error] Prerelease not allow for builds that put build number as the patch version" - # exit -1; - # fi - # fi - - # if [[ "$(release)" != "prerelease" ]]; then - # if [[ "${{ parameters.packageTypesOverride }}" == "alpha" || "${{ parameters.packageTypesOverride }}" == "beta" ]]; then - # echo "##vso[task.logissue type=error]This release type is not supported. alpha/beta ***prerelease*** is allowed" - # exit -1; - # fi - # fi - - # if [[ "$(release)" != "none" ]] && [[ "$(release)" != "" ]]; then - # if [[ "${{ parameters.publish }}" != "True" ]]; then - # echo "##vso[task.logissue type=error]'$(release)'' is set but package is not published. Either the branch doesn't default to publish or it is skipped." - # exit -1; - # fi - # fi - - # - template: /tools/pipelines/templates/include-use-node-version.yml@self - - # - template: /tools/pipelines/templates/include-install.yml@self - # parameters: - # packageManager: '${{ parameters.packageManager }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - # packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' - - # # This check is a workaround. We don't want to set versions for the build-bundle-size-and-code-coverage-artifacts - # # pipeline because it is special - it runs a client build but doesn't publish anything. Working around this properly is - # # challenging and would create a much bigger change. Since this is the only pipeline that sets these variables to - # # true, we use that to determine whether to set versions. - # - ${{ if eq(parameters.taskPublishBundleSizeArtifacts, false) }}: - # - template: /tools/pipelines/templates/include-set-package-version.yml@self - # parameters: - # buildDirectory: '${{ parameters.buildDirectory }}' - # buildNumberInPatch: ${{ parameters.buildNumberInPatch }} - # buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}' - # tagName: '${{ parameters.tagName }}' - # interdependencyRange: '${{ parameters.interdependencyRange }}' - # packageTypesOverride: '${{ parameters.packageTypesOverride }}' - - # # Build and Lint - # - template: /tools/pipelines/templates/include-build-lint.yml@self - # parameters: - # taskBuild: '${{ parameters.taskBuild }}' - # taskLint: '${{ parameters.taskLint }}' - # taskLintName: '${{ parameters.taskLintName }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - - # - task: Npm@1 - # displayName: 'npm run webpack' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run webpack' - - # - task: Bash@3 - # displayName: Archive Nested Lib/Dist Directories and Their Contents - # env: - # WORKING_DIRECTORY: '${{ parameters.buildDirectory }}' - # inputs: - # targetType: filePath - # workingDirectory: '${{ parameters.buildDirectory }}' - # filePath: $(Build.SourcesDirectory)/scripts/pack-distlib.sh - - # - task: CopyFiles@2 - # displayName: Copy nested_lib_dist to artifact staging directory - # inputs: - # sourceFolder: ${{ parameters.buildDirectory }}/nested_lib_dist - # targetFolder: $(Build.ArtifactStagingDirectory)/nested_lib_dist - - # # Pack - # - ${{ if ne(parameters.taskPack, false) }}: - # - task: Bash@3 - # displayName: npm pack - # env: - # PACKAGE_MANAGER: '${{ parameters.packageManager }}' - # RELEASE_GROUP: '${{ parameters.tagName }}' - # STAGING_PATH: $(Build.ArtifactStagingDirectory) - # inputs: - # targetType: filePath - # workingDirectory: '${{ parameters.buildDirectory }}' - # filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh - - # # At this point we want to publish the artifact with npm-packed packages, and the one with test files, - # # but as part of 1ES migration that's now part of templateContext.outputs below. - - # # Collect/publish/run bundle analysis - # - ${{ if eq(parameters.taskBundleAnalysis, true) }}: - # - task: Npm@1 - # displayName: 'npm run bundle-analysis:collect' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run bundle-analysis:collect' - - # # Copy files so all artifacts we publish end up under the same parent folder. - # # The sourceFolder should be wherever the 'npm run bundle-analysis:collect' task places its output. - # - task: CopyFiles@2 - # displayName: Copy bundleAnalysis files to artifact staging directory - # inputs: - # sourceFolder: '${{ parameters.buildDirectory }}/artifacts/bundleAnalysis' - # targetFolder: $(Build.ArtifactStagingDirectory)/bundleAnalysis - - - # # At this point we want to publish the artifact with the bundle size analysis, - # # but as part of 1ES migration that's now part of templateContext.outputs below. - - # - task: Npm@1 - # displayName: run bundle size comparison - # condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - # continueOnError: true - # env: - # ADO_API_TOKEN: $(System.AccessToken) - # DANGER_GITHUB_API_TOKEN: $(githubPublicRepoSecret) - # TARGET_BRANCH_NAME: '$(targetBranchName)' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run bundle-analysis:run' - - # - ${{ if and(or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')), eq(variables['System.TeamProject'], 'internal')) }}: - # - task: Bash@3 - # displayName: List report.json - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # echo "Build Directory is ${{ parameters.buildDirectory }}"; - # BUNDLE_SIZE_TESTS_DIR="$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests"; - # echo "Contents of $BUNDLE_SIZE_TESTS_DIR:"; - # ls -la $BUNDLE_SIZE_TESTS_DIR; - - # - template: /tools/pipelines/templates/include-telemetry-setup.yml@self - # parameters: - # pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) - - # - task: Bash@3 - # displayName: Write bundle sizes measurements to Aria/Kusto - # inputs: - # targetType: inline - # workingDirectory: $(pathToTelemetryGenerator) - # script: | - # set -eu -o pipefail - # echo "Writing the following performance tests results to Aria/Kusto" - # echo "Report Size:" - # ls -la '$(Build.SourcesDirectory)/examples/utils/bundle-size-tests/bundleAnalysis/report.json'; - # npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/bundleSizeHandler.js" --dir '$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests'; - - # # Docs - # - ${{ if ne(parameters.taskBuildDocs, false) }}: - # - task: Npm@1 - # displayName: 'npm run ci:build:docs' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run ci:build:docs' - - # # Copy files so all artifacts we publish end up under the same parent folder. - # # The sourceFolder should be wherever the 'npm run ci:build:docs' task places its output. - # - task: CopyFiles@2 - # displayName: Copy _api-extractor-temp files to artifact staging directory - # inputs: - # sourceFolder: '${{ parameters.buildDirectory }}/_api-extractor-temp' - # targetFolder: $(Build.ArtifactStagingDirectory)/_api-extractor-temp - - # # At this point we want to publish the artifact with the _api-extractor-temp folder, - # # but as part of 1ES migration that's now part of templateContext.outputs below. - - # - ${{ if eq(parameters.packageManager, 'pnpm') }}: - # # Reset the pnpm-lock.yaml file since it's been modified by the versioning. But for dependency caching we want - # # the cache key (which is based on the contents of the lockfile) to be the unmodified file. So we reset the - # # lockfile as the last step so that when the dependency cache is uploaded, the cache key matches what it was - # # at the beginning of the CI job. - # - task: Bash@3 - # displayName: Reset lockfile - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # git checkout HEAD -- pnpm-lock.yaml - - # # Prune the pnpm store before it's cached. This removes any deps that are not used by the current build. - # - task: Bash@3 - # displayName: Prune pnpm store - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # pnpm store prune - - # - task: Bash@3 - # displayName: Check for extraneous modified files - # inputs: - # targetType: inline - # script: | - # # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep - # # even in an error case - # git status | grep -v -E 'package.json|package-lock.json|packageVersion.ts|lerna.json|.npmrc|build-tools/.npmrc|\(use.*' | grep '^\s' > git_status.log - # if [ `cat git_status.log | wc -l` != "0" ]; then - # cat git_status.log - # echo "##vso[task.logissue type=error]Build should not create extraneous files" - # exit -1; - # fi - - # # This additional build step is used to run step not part of the main build. In build client pipeline, - # # this is used to inject telemetry key file and upload built devtools extension. - # - ${{ parameters.additionalBuildSteps }} - - # templateContext: - # outputParentDirectory: $(Build.ArtifactStagingDirectory) - # outputs: - # - output: pipelineArtifact - # displayName: Publish Artifact - nested_lib_dist - # targetPath: $(Build.ArtifactStagingDirectory)/nested_lib_dist - # artifactName: nested_lib_dist - # publishLocation: pipeline - - # - ${{ if ne(parameters.taskPack, false) }}: - # - output: pipelineArtifact - # displayName: Publish Artifact - pack - # targetPath: $(Build.ArtifactStagingDirectory)/pack - # artifactName: pack - # publishLocation: pipeline - - # - output: pipelineArtifact - # displayName: Publish Artifact - Test Files - # targetPath: $(Build.ArtifactStagingDirectory)/test-files - # artifactName: test-files - # publishLocation: pipeline - # sbomEnabled: false - - # - ${{ if eq(parameters.taskBundleAnalysis, true) }}: - # - output: pipelineArtifact - # displayName: Publish Artifacts - bundle-analysis - # condition: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(${{ parameters.taskPublishBundleSizeArtifacts }}, true) ) - # targetPath: $(Build.ArtifactStagingDirectory)/bundleAnalysis - # artifactName: bundleAnalysis - # sbomEnabled: false - # publishLocation: pipeline - - # - ${{ if or(eq(parameters.publishDocs, true), eq(parameters.taskBuildDocs, true)) }}: - # - output: pipelineArtifact - # displayName: Publish Artifact - _api-extractor-temp - # targetPath: $(Build.ArtifactStagingDirectory)/_api-extractor-temp - # artifactName: _api-extractor-temp - # sbomEnabled: false - # publishLocation: pipeline - - # - job: Coverage_tests - # displayName: "Coverage tests" - # dependsOn: build - # variables: - # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # - name: targetBranchName - # value: $(System.PullRequest.TargetBranch) - # # Absolute path to the folder that contains the source code for the telemetry-generator package, which is - # # used in a few places in the pipeline to push custom telemetry to Kusto. - # - name: absolutePathToTelemetryGenerator - # value: $(Build.SourcesDirectory)/tools/telemetry-generator - # readonly: true - # # We already run CodeQL in the main build job, so we don't need to run it again here. - # # Note that we need to disable it in the right way for 1ES pipeline templates, vs manual CodeQL tasks. - # - name: ONEES_ENFORCED_CODEQL_ENABLED - # value: 'false' - # steps: - # # Setup - # - checkout: self - # clean: true - # lfs: '${{ parameters.checkoutSubmodules }}' - # submodules: '${{ parameters.checkoutSubmodules }}' - - # - template: /tools/pipelines/templates/include-use-node-version.yml@self - - # - template: /tools/pipelines/templates/include-install.yml@self - # parameters: - # packageManager: '${{ parameters.packageManager }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - # packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' - - # # We need it in order to run flub where the code coverage comparison logic calls for it - # - template: /tools/pipelines/templates/include-install-build-tools.yml@self - # parameters: - # buildDirectory: ${{ parameters.buildDirectory }} - # buildToolsVersionToInstall: repo - # pnpmStorePath: $(Pipeline.Workspace)/.pnpm-store - - # - task: DownloadPipelineArtifact@2 - # inputs: - # artifact: nested_lib_dist - # targetPath: $(Build.SourcesDirectory) - - # - script: | - # echo "Extracting nested lib/dist directories and their contents..." - # tar --extract --gzip --file $(Build.SourcesDirectory)/nested_lib_dist.tar.gz --directory $(Build.SourcesDirectory) - # displayName: Extract Nested Lib/Dist Directories and Their Contents - - # # Set variable startTest if everything is good so far and we'll start running tests, - # # so that the steps to process/upload test coverage results only run if we got to the point of actually running tests. - # - script: | - # echo "##vso[task.setvariable variable=startTest]true" - # displayName: Start Test - - # - ${{ each test in parameters.coverageTests }}: - # - template: /tools/pipelines/templates/include-test-task.yml@self - # parameters: - # taskTestStep: '${{ test.name }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - # testCoverage: '${{ parameters.testCoverage }}' - - # - task: Npm@1 - # displayName: 'npm run test:copyresults' - # condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run test:copyresults' - - # # Test - Upload coverage results - # # Some webpacked file using externals introduce file name with quotes in them - # # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish - # # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) - # - task: Bash@3 - # displayName: Check for nyc/report directory - # condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) - # inputs: - # targetType: 'inline' - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' - - # - task: Bash@3 - # displayName: Patch Coverage Results - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # targetType: 'inline' - # workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' - # script: | - # set -eu -o pipefail - # sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml - - # - task: PublishCodeCoverageResults@2 - # displayName: Publish Code Coverage - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml - # failIfCoverageEmpty: true - # - task: CopyFiles@2 - # displayName: Copy code coverage report to artifact staging directory - # condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # inputs: - # sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' - # targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - # - task: Bash@3 - # displayName: Report Code Coverage Comparison - # condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) - # continueOnError: false - # env: - # ADO_API_TOKEN: '$(System.AccessToken)' - # GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' - # TARGET_BRANCH_NAME: '$(targetBranchName)' - # ADO_BUILD_ID: '$(Build.BuildId)' - # GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' - # GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' - # ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 - # ADO_CI_BUILD_DEFINITION_ID_PR: 11 - # inputs: - # targetType: inline - # workingDirectory: '${{ parameters.buildDirectory }}' - # script: | - # set -eu -o pipefail - # echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" - # echo "Github PR number: $GITHUB_PR_NUMBER" - # echo "ADO Build Number: $ADO_BUILD_ID" - # echo "Target Branch Name: $TARGET_BRANCH_NAME" - # echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" - # echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" - # echo "Running code coverage comparison" - # flub report codeCoverage --verbose - - # # Process test result, include publishing and logging - # - template: /tools/pipelines/templates/include-process-test-results.yml@self - # parameters: - # buildDirectory: '${{ parameters.buildDirectory }}' - # testResultDirs: '${{ parameters.testResultDirs }}' - - # templateContext: - # outputParentDirectory: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - # outputs: - # - output: pipelineArtifact - # displayName: Publish Artifacts - code-coverage - # condition: and( succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) - # targetPath: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis - # artifactName: 'codeCoverageAnalysis-$(System.JobAttempt)' - # sbomEnabled: false - # publishLocation: pipeline - - # # Parallel jobs for test tasks - # - ${{ each test in parameters.taskTest }}: - # - job: Test_${{ test.jobName }} - # displayName: "Run Task Test ${{ test.jobName }}" - # dependsOn: build - # variables: - # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # - name: targetBranchName - # value: $(System.PullRequest.TargetBranch) - # # Absolute path to the folder that contains the source code for the telemetry-generator package, which is - # # used in a few places in the pipeline to push custom telemetry to Kusto. - # - name: absolutePathToTelemetryGenerator - # value: $(Build.SourcesDirectory)/tools/telemetry-generator - # readonly: true - # # We already run CodeQL in the main build job, so we don't need to run it again here. - # # Note that we need to disable it in the right way for 1ES pipeline templates, vs manual CodeQL tasks. - # - name: ONEES_ENFORCED_CODEQL_ENABLED - # value: 'false' - # steps: - # # Setup - # - checkout: self - # clean: true - # lfs: '${{ parameters.checkoutSubmodules }}' - # submodules: '${{ parameters.checkoutSubmodules }}' - - # - template: /tools/pipelines/templates/include-use-node-version.yml@self - - # - template: /tools/pipelines/templates/include-install.yml@self - # parameters: - # packageManager: '${{ parameters.packageManager }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - # packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' - - # - task: DownloadPipelineArtifact@2 - # inputs: - # artifact: nested_lib_dist - # targetPath: $(Build.SourcesDirectory) - - # - script: | - # echo "Extracting nested lib/dist directories and their contents..." - # tar --extract --gzip --file $(Build.SourcesDirectory)/nested_lib_dist.tar.gz --directory $(Build.SourcesDirectory) - # displayName: Extract Nested Lib/Dist Directories and Their Contents - - # # Test - - # - ${{ if contains(test.name, 'jest') }}: - # - task: Npm@1 - # displayName: 'Run build for jest tests' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run ci:build' - - # # Set variable startTest if everything is good so far and we'll start running tests, - # # so that the steps to process/upload test coverage results only run if we got to the point of actually running tests. - # - script: | - # echo "##vso[task.setvariable variable=startTest]true" - # displayName: Start Test - - # - template: /tools/pipelines/templates/include-test-task.yml@self - # parameters: - # taskTestStep: '${{ test.name }}' - # buildDirectory: '${{ parameters.buildDirectory }}' - # testCoverage: 'false' - - # - task: Npm@1 - # displayName: 'npm run test:copyresults' - # inputs: - # command: custom - # workingDir: '${{ parameters.buildDirectory }}' - # customCommand: 'run test:copyresults' - - # - ${{ if contains(test.name, 'tinylicious') }}: - # - task: Bash@3 - # displayName: Upload tinylicious log - # condition: always() - # continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) - # inputs: - # targetType: inline - # script: | - # set -eu -o pipefail - # PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; - # if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then - # echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; - # echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; - # else - # echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; - # fi - - # # Process test result, include publishing and logging - # - template: /tools/pipelines/templates/include-process-test-results.yml@self - # parameters: - # buildDirectory: '${{ parameters.buildDirectory }}' - # testResultDirs: '${{ parameters.testResultDirs }}' + - stage: build + displayName: Build Stage + dependsOn: [] # this stage doesn't depend on preceding stage + jobs: + # Job - Build + - job: build + displayName: Build + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + timeoutInMinutes: 120 + ${{ else }}: + # CI builds run more aggressive compat configurations which can take longer. + # See "FullCompat" under packages\test\test-version-utils\README.md for more details. + # At the time of adding this comment, the full compat config is on the smaller side and so + # CI builds consistently pass with a 60 minutes timeout. However, it will naturally grow + # over time and it might be necessary to bump it. + # AB#6680 is also relevant here, which tracks rethinking how and where we run tests (likely with + # a focus on e2e tests) + # Note, This was recently updated to 90 minutes to account for the additional build time added from extending + # the Microsoft 1ES template required for corporate security compliance. Updated again to 120 to mitigate a + # series of build breaks due to timeouts. + timeoutInMinutes: 120 + variables: + - group: ado-feeds + - group: storage-vars + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - name: targetBranchName + value: $(System.PullRequest.TargetBranch) + steps: + # Setup + - checkout: self + clean: true + lfs: '${{ parameters.checkoutSubmodules }}' + submodules: '${{ parameters.checkoutSubmodules }}' + + - task: Bash@3 + displayName: Parameters + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep + # even in an error case + + # Show all task group conditions + + echo " + Pipeline Variables: + + Override Parameters: + packageTypesOverride=${{ parameters.packageTypesOverride }} + publishOverride=${{ parameters.publishOverride }} + releaseBuildOverride=${{ parameters.releaseBuildOverride }} + + Tasks Parameters: + Build=${{ parameters.taskBuild }} + BuildDir=${{ parameters.buildDirectory }} + BuildDoc=${{ parameters.taskBuildDocs }} + Lint=${{ parameters.taskLint }} + LintName: ${{ parameters.taskLintName }} + PublishDocs=${{ parameters.publishDocs }} + Test=${{ convertToJson(parameters.taskTest) }} + TestCoverage=${{ parameters.testCoverage }} + TestResultDirs=${{ convertToJson(parameters.testResultDirs) }} + + Variables: + pathToTelemetryGenerator=$(pathToTelemetryGenerator) + BuildReason=${{ variables['Build.Reason'] }} + + Publish Parameters: + interdependencyRange='${{ parameters.interdependencyRange }}' + packageTypesOverride='${{ parameters.packageTypesOverride }}' + publish=${{ parameters.publish }} + + Computed variables: + canRelease=$(canRelease) + release=$(release) + shouldPublish=$(shouldPublish) + " + + # Target Branch variable (PR policy related) + if [[ ${{ variables['Build.Reason'] }} == "PullRequest" ]]; then + echo "TargetBranchName=$(targetBranchName)" + fi + + # Error checking + if [[ "$(release)" == "release" ]]; then + if [[ "$(canRelease)" == "False" ]]; then + echo "##vso[task.logissue type=error]Invalid branch ${{ variables['Build.SourceBranch'] }} for release" + exit -1; + fi + + if [ -f ".releaseGroup" ]; then + grep -e fluid.*[0-9]-[0-9] `find packages -name 'package.json'` + else + grep -e fluid.*[0-9]-[0-9] `find . -name 'package.json'` + fi + + if [[ $? == 0 ]]; then + echo "##vso[task.logissue type=error]Release shouldn't contain prerelease dependencies" + exit -1; + fi + fi + + if [[ "$(release)" == "prerelease" ]]; then + if [[ "${{ parameters.buildNumberInPatch }}" == "true" ]]; then + echo "##vso[task.logissue type=error] Prerelease not allow for builds that put build number as the patch version" + exit -1; + fi + fi + + if [[ "$(release)" != "prerelease" ]]; then + if [[ "${{ parameters.packageTypesOverride }}" == "alpha" || "${{ parameters.packageTypesOverride }}" == "beta" ]]; then + echo "##vso[task.logissue type=error]This release type is not supported. alpha/beta ***prerelease*** is allowed" + exit -1; + fi + fi + + if [[ "$(release)" != "none" ]] && [[ "$(release)" != "" ]]; then + if [[ "${{ parameters.publish }}" != "True" ]]; then + echo "##vso[task.logissue type=error]'$(release)'' is set but package is not published. Either the branch doesn't default to publish or it is skipped." + exit -1; + fi + fi + + - template: /tools/pipelines/templates/include-use-node-version.yml@self + + - template: /tools/pipelines/templates/include-install.yml@self + parameters: + packageManager: '${{ parameters.packageManager }}' + buildDirectory: '${{ parameters.buildDirectory }}' + packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' + + # This check is a workaround. We don't want to set versions for the build-bundle-size-and-code-coverage-artifacts + # pipeline because it is special - it runs a client build but doesn't publish anything. Working around this properly is + # challenging and would create a much bigger change. Since this is the only pipeline that sets these variables to + # true, we use that to determine whether to set versions. + - ${{ if eq(parameters.taskPublishBundleSizeArtifacts, false) }}: + - template: /tools/pipelines/templates/include-set-package-version.yml@self + parameters: + buildDirectory: '${{ parameters.buildDirectory }}' + buildNumberInPatch: ${{ parameters.buildNumberInPatch }} + buildToolsVersionToInstall: '${{ parameters.buildToolsVersionToInstall }}' + tagName: '${{ parameters.tagName }}' + interdependencyRange: '${{ parameters.interdependencyRange }}' + packageTypesOverride: '${{ parameters.packageTypesOverride }}' + + # Build and Lint + - template: /tools/pipelines/templates/include-build-lint.yml@self + parameters: + taskBuild: '${{ parameters.taskBuild }}' + taskLint: '${{ parameters.taskLint }}' + taskLintName: '${{ parameters.taskLintName }}' + buildDirectory: '${{ parameters.buildDirectory }}' + + - task: Npm@1 + displayName: 'npm run webpack' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run webpack' + + - task: Bash@3 + displayName: Archive Nested Lib/Dist Directories and Their Contents + env: + WORKING_DIRECTORY: '${{ parameters.buildDirectory }}' + inputs: + targetType: filePath + workingDirectory: '${{ parameters.buildDirectory }}' + filePath: $(Build.SourcesDirectory)/scripts/pack-distlib.sh + + - task: CopyFiles@2 + displayName: Copy nested_lib_dist to artifact staging directory + inputs: + sourceFolder: ${{ parameters.buildDirectory }}/nested_lib_dist + targetFolder: $(Build.ArtifactStagingDirectory)/nested_lib_dist + + # Pack + - ${{ if ne(parameters.taskPack, false) }}: + - task: Bash@3 + displayName: npm pack + env: + PACKAGE_MANAGER: '${{ parameters.packageManager }}' + RELEASE_GROUP: '${{ parameters.tagName }}' + STAGING_PATH: $(Build.ArtifactStagingDirectory) + inputs: + targetType: filePath + workingDirectory: '${{ parameters.buildDirectory }}' + filePath: $(Build.SourcesDirectory)/scripts/pack-packages.sh + + # At this point we want to publish the artifact with npm-packed packages, and the one with test files, + # but as part of 1ES migration that's now part of templateContext.outputs below. + + # Collect/publish/run bundle analysis + - ${{ if eq(parameters.taskBundleAnalysis, true) }}: + - task: Npm@1 + displayName: 'npm run bundle-analysis:collect' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run bundle-analysis:collect' + + # Copy files so all artifacts we publish end up under the same parent folder. + # The sourceFolder should be wherever the 'npm run bundle-analysis:collect' task places its output. + - task: CopyFiles@2 + displayName: Copy bundleAnalysis files to artifact staging directory + inputs: + sourceFolder: '${{ parameters.buildDirectory }}/artifacts/bundleAnalysis' + targetFolder: $(Build.ArtifactStagingDirectory)/bundleAnalysis + + + # At this point we want to publish the artifact with the bundle size analysis, + # but as part of 1ES migration that's now part of templateContext.outputs below. + + - task: Npm@1 + displayName: run bundle size comparison + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + continueOnError: true + env: + ADO_API_TOKEN: $(System.AccessToken) + DANGER_GITHUB_API_TOKEN: $(githubPublicRepoSecret) + TARGET_BRANCH_NAME: '$(targetBranchName)' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run bundle-analysis:run' + + - ${{ if and(or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')), eq(variables['System.TeamProject'], 'internal')) }}: + - task: Bash@3 + displayName: List report.json + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + echo "Build Directory is ${{ parameters.buildDirectory }}"; + BUNDLE_SIZE_TESTS_DIR="$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests"; + echo "Contents of $BUNDLE_SIZE_TESTS_DIR:"; + ls -la $BUNDLE_SIZE_TESTS_DIR; + + - template: /tools/pipelines/templates/include-telemetry-setup.yml@self + parameters: + pathForTelemetryGeneratorInstall: $(pathToTelemetryGenerator) + + - task: Bash@3 + displayName: Write bundle sizes measurements to Aria/Kusto + inputs: + targetType: inline + workingDirectory: $(pathToTelemetryGenerator) + script: | + set -eu -o pipefail + echo "Writing the following performance tests results to Aria/Kusto" + echo "Report Size:" + ls -la '$(Build.SourcesDirectory)/examples/utils/bundle-size-tests/bundleAnalysis/report.json'; + npx telemetry-generator --handlerModule "$(pathToTelemetryGeneratorHandlers)/bundleSizeHandler.js" --dir '$(Build.ArtifactStagingDirectory)/bundleAnalysis/@fluid-example/bundle-size-tests'; + + # Docs + - ${{ if ne(parameters.taskBuildDocs, false) }}: + - task: Npm@1 + displayName: 'npm run ci:build:docs' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run ci:build:docs' + + # Copy files so all artifacts we publish end up under the same parent folder. + # The sourceFolder should be wherever the 'npm run ci:build:docs' task places its output. + - task: CopyFiles@2 + displayName: Copy _api-extractor-temp files to artifact staging directory + inputs: + sourceFolder: '${{ parameters.buildDirectory }}/_api-extractor-temp' + targetFolder: $(Build.ArtifactStagingDirectory)/_api-extractor-temp + + # At this point we want to publish the artifact with the _api-extractor-temp folder, + # but as part of 1ES migration that's now part of templateContext.outputs below. + + - ${{ if eq(parameters.packageManager, 'pnpm') }}: + # Reset the pnpm-lock.yaml file since it's been modified by the versioning. But for dependency caching we want + # the cache key (which is based on the contents of the lockfile) to be the unmodified file. So we reset the + # lockfile as the last step so that when the dependency cache is uploaded, the cache key matches what it was + # at the beginning of the CI job. + - task: Bash@3 + displayName: Reset lockfile + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + git checkout HEAD -- pnpm-lock.yaml + + # Prune the pnpm store before it's cached. This removes any deps that are not used by the current build. + - task: Bash@3 + displayName: Prune pnpm store + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + pnpm store prune + + - task: Bash@3 + displayName: Check for extraneous modified files + inputs: + targetType: inline + script: | + # Note: deliberately not using `set -eu -o pipefail` because this script leverages the return code of grep + # even in an error case + git status | grep -v -E 'package.json|package-lock.json|packageVersion.ts|lerna.json|.npmrc|build-tools/.npmrc|\(use.*' | grep '^\s' > git_status.log + if [ `cat git_status.log | wc -l` != "0" ]; then + cat git_status.log + echo "##vso[task.logissue type=error]Build should not create extraneous files" + exit -1; + fi + + # This additional build step is used to run step not part of the main build. In build client pipeline, + # this is used to inject telemetry key file and upload built devtools extension. + - ${{ parameters.additionalBuildSteps }} + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + displayName: Publish Artifact - nested_lib_dist + targetPath: $(Build.ArtifactStagingDirectory)/nested_lib_dist + artifactName: nested_lib_dist + publishLocation: pipeline + + - ${{ if ne(parameters.taskPack, false) }}: + - output: pipelineArtifact + displayName: Publish Artifact - pack + targetPath: $(Build.ArtifactStagingDirectory)/pack + artifactName: pack + publishLocation: pipeline + + - output: pipelineArtifact + displayName: Publish Artifact - Test Files + targetPath: $(Build.ArtifactStagingDirectory)/test-files + artifactName: test-files + publishLocation: pipeline + sbomEnabled: false + + - ${{ if eq(parameters.taskBundleAnalysis, true) }}: + - output: pipelineArtifact + displayName: Publish Artifacts - bundle-analysis + condition: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(${{ parameters.taskPublishBundleSizeArtifacts }}, true) ) + targetPath: $(Build.ArtifactStagingDirectory)/bundleAnalysis + artifactName: bundleAnalysis + sbomEnabled: false + publishLocation: pipeline + + - ${{ if or(eq(parameters.publishDocs, true), eq(parameters.taskBuildDocs, true)) }}: + - output: pipelineArtifact + displayName: Publish Artifact - _api-extractor-temp + targetPath: $(Build.ArtifactStagingDirectory)/_api-extractor-temp + artifactName: _api-extractor-temp + sbomEnabled: false + publishLocation: pipeline + + - job: Coverage_tests + displayName: "Coverage tests" + dependsOn: build + variables: + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - name: targetBranchName + value: $(System.PullRequest.TargetBranch) + # Absolute path to the folder that contains the source code for the telemetry-generator package, which is + # used in a few places in the pipeline to push custom telemetry to Kusto. + - name: absolutePathToTelemetryGenerator + value: $(Build.SourcesDirectory)/tools/telemetry-generator + readonly: true + # We already run CodeQL in the main build job, so we don't need to run it again here. + # Note that we need to disable it in the right way for 1ES pipeline templates, vs manual CodeQL tasks. + - name: ONEES_ENFORCED_CODEQL_ENABLED + value: 'false' + steps: + # Setup + - checkout: self + clean: true + lfs: '${{ parameters.checkoutSubmodules }}' + submodules: '${{ parameters.checkoutSubmodules }}' + + - template: /tools/pipelines/templates/include-use-node-version.yml@self + + - template: /tools/pipelines/templates/include-install.yml@self + parameters: + packageManager: '${{ parameters.packageManager }}' + buildDirectory: '${{ parameters.buildDirectory }}' + packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' + + # We need it in order to run flub where the code coverage comparison logic calls for it + - template: /tools/pipelines/templates/include-install-build-tools.yml@self + parameters: + buildDirectory: ${{ parameters.buildDirectory }} + buildToolsVersionToInstall: repo + pnpmStorePath: $(Pipeline.Workspace)/.pnpm-store + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: nested_lib_dist + targetPath: $(Build.SourcesDirectory) + + - script: | + echo "Extracting nested lib/dist directories and their contents..." + tar --extract --gzip --file $(Build.SourcesDirectory)/nested_lib_dist.tar.gz --directory $(Build.SourcesDirectory) + displayName: Extract Nested Lib/Dist Directories and Their Contents + + # Set variable startTest if everything is good so far and we'll start running tests, + # so that the steps to process/upload test coverage results only run if we got to the point of actually running tests. + - script: | + echo "##vso[task.setvariable variable=startTest]true" + displayName: Start Test + + - ${{ each test in parameters.coverageTests }}: + - template: /tools/pipelines/templates/include-test-task.yml@self + parameters: + taskTestStep: '${{ test.name }}' + buildDirectory: '${{ parameters.buildDirectory }}' + testCoverage: '${{ parameters.testCoverage }}' + + - task: Npm@1 + displayName: 'npm run test:copyresults' + condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run test:copyresults' + + # Test - Upload coverage results + # Some webpacked file using externals introduce file name with quotes in them + # and Istanbul's cobertura reporter doesn't escape them causing parse error when we publish + # A quick fix to patch the file with sed. (See https://github.com/bcoe/c8/issues/302) + - task: Bash@3 + displayName: Check for nyc/report directory + condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) + inputs: + targetType: 'inline' + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + test -d nyc/report && echo '##vso[task.setvariable variable=ReportDirExists;]true' || echo 'No nyc/report directory' + + - task: Bash@3 + displayName: Patch Coverage Results + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + targetType: 'inline' + workingDirectory: '${{ parameters.buildDirectory }}/nyc/report' + script: | + set -eu -o pipefail + sed -e 's/\(filename=\".*[\\/]external .*\)"\(.*\)""/\1\"\2\""/' cobertura-coverage.xml > cobertura-coverage-patched.xml + + - task: PublishCodeCoverageResults@2 + displayName: Publish Code Coverage + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + summaryFileLocation: ${{ parameters.buildDirectory }}/nyc/report/cobertura-coverage-patched.xml + failIfCoverageEmpty: true + - task: CopyFiles@2 + displayName: Copy code coverage report to artifact staging directory + condition: and(succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + inputs: + sourceFolder: '${{ parameters.buildDirectory }}/nyc/report' + targetFolder: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + - task: Bash@3 + displayName: Report Code Coverage Comparison + condition: and(succeededOrFailed(), eq('${{ parameters.reportCodeCoverageComparison }}', true), eq(variables['ReportDirExists'], 'true'), eq(variables['System.PullRequest.TargetBranch'], 'main')) + continueOnError: false + env: + ADO_API_TOKEN: '$(System.AccessToken)' + GITHUB_API_TOKEN: '$(githubPublicRepoSecret)' + TARGET_BRANCH_NAME: '$(targetBranchName)' + ADO_BUILD_ID: '$(Build.BuildId)' + GITHUB_PR_NUMBER: '$(System.PullRequest.PullRequestNumber)' + GITHUB_REPOSITORY_NAME: '$(Build.Repository.Name)' + ADO_CI_BUILD_DEFINITION_ID_BASELINE: 48 + ADO_CI_BUILD_DEFINITION_ID_PR: 11 + inputs: + targetType: inline + workingDirectory: '${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + echo "Github Repository Name: $GITHUB_REPOSITORY_NAME" + echo "Github PR number: $GITHUB_PR_NUMBER" + echo "ADO Build Number: $ADO_BUILD_ID" + echo "Target Branch Name: $TARGET_BRANCH_NAME" + echo "ADO CI BUILD_DEFINITION_ID for baseline: $ADO_CI_BUILD_DEFINITION_ID_BASELINE" + echo "ADO CI BUILD_DEFINITION_ID for PR: $ADO_CI_BUILD_DEFINITION_ID_PR" + echo "Running code coverage comparison" + flub report codeCoverage --verbose + + # Process test result, include publishing and logging + - template: /tools/pipelines/templates/include-process-test-results.yml@self + parameters: + buildDirectory: '${{ parameters.buildDirectory }}' + testResultDirs: '${{ parameters.testResultDirs }}' + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + outputs: + - output: pipelineArtifact + displayName: Publish Artifacts - code-coverage + condition: and( succeededOrFailed(), eq(variables['ReportDirExists'], 'true')) + targetPath: $(Build.ArtifactStagingDirectory)/codeCoverageAnalysis + artifactName: 'codeCoverageAnalysis-$(System.JobAttempt)' + sbomEnabled: false + publishLocation: pipeline + + # Parallel jobs for test tasks + - ${{ each test in parameters.taskTest }}: + - job: Test_${{ test.jobName }} + displayName: "Run Task Test ${{ test.jobName }}" + dependsOn: build + variables: + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - name: targetBranchName + value: $(System.PullRequest.TargetBranch) + # Absolute path to the folder that contains the source code for the telemetry-generator package, which is + # used in a few places in the pipeline to push custom telemetry to Kusto. + - name: absolutePathToTelemetryGenerator + value: $(Build.SourcesDirectory)/tools/telemetry-generator + readonly: true + # We already run CodeQL in the main build job, so we don't need to run it again here. + # Note that we need to disable it in the right way for 1ES pipeline templates, vs manual CodeQL tasks. + - name: ONEES_ENFORCED_CODEQL_ENABLED + value: 'false' + steps: + # Setup + - checkout: self + clean: true + lfs: '${{ parameters.checkoutSubmodules }}' + submodules: '${{ parameters.checkoutSubmodules }}' + + - template: /tools/pipelines/templates/include-use-node-version.yml@self + + - template: /tools/pipelines/templates/include-install.yml@self + parameters: + packageManager: '${{ parameters.packageManager }}' + buildDirectory: '${{ parameters.buildDirectory }}' + packageManagerInstallCommand: '${{ parameters.packageManagerInstallCommand }}' + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: nested_lib_dist + targetPath: $(Build.SourcesDirectory) + + - script: | + echo "Extracting nested lib/dist directories and their contents..." + tar --extract --gzip --file $(Build.SourcesDirectory)/nested_lib_dist.tar.gz --directory $(Build.SourcesDirectory) + displayName: Extract Nested Lib/Dist Directories and Their Contents + + # Test + + - ${{ if contains(test.name, 'jest') }}: + - task: Npm@1 + displayName: 'Run build for jest tests' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run ci:build' + + # Set variable startTest if everything is good so far and we'll start running tests, + # so that the steps to process/upload test coverage results only run if we got to the point of actually running tests. + - script: | + echo "##vso[task.setvariable variable=startTest]true" + displayName: Start Test + + - template: /tools/pipelines/templates/include-test-task.yml@self + parameters: + taskTestStep: '${{ test.name }}' + buildDirectory: '${{ parameters.buildDirectory }}' + testCoverage: 'false' + + - task: Npm@1 + displayName: 'npm run test:copyresults' + inputs: + command: custom + workingDir: '${{ parameters.buildDirectory }}' + customCommand: 'run test:copyresults' + + - ${{ if contains(test.name, 'tinylicious') }}: + - task: Bash@3 + displayName: Upload tinylicious log + condition: always() + continueOnError: true # Keep running subsequent tasks even if this one fails (e.g. the tinylicious log wasn't there) + inputs: + targetType: inline + script: | + set -eu -o pipefail + PATH_TO_TINYLICIOUS_LOG=$(Build.SourcesDirectory)/packages/test/test-end-to-end-tests/tinylicious.log; + if [ -f $PATH_TO_TINYLICIOUS_LOG ] ; then + echo "Found file at '$PATH_TO_TINYLICIOUS_LOG'. Uploading."; + echo "##vso[task.uploadfile]$PATH_TO_TINYLICIOUS_LOG"; + else + echo "##vso[task.logissue type=warning]Failed to upload tinylicious log file ('$PATH_TO_TINYLICIOUS_LOG' not found)."; + fi + + # Process test result, include publishing and logging + - template: /tools/pipelines/templates/include-process-test-results.yml@self + parameters: + buildDirectory: '${{ parameters.buildDirectory }}' + testResultDirs: '${{ parameters.testResultDirs }}' # # Publish stage # - ${{ if eq(parameters.publish, true) }}: From ff323552e8c4e06b8d304c9a7daf4bb47f89800c Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:10:08 +0000 Subject: [PATCH 55/58] Attempt to use a test tempalted deployment job without extensive build stage changes --- .../templates/build-npm-client-package.yml | 37 ++++---- .../pipelines/templates/build-npm-package.yml | 6 ++ .../test-template-deployment-job.yml | 94 +++++++++++++++++++ 3 files changed, 118 insertions(+), 19 deletions(-) create mode 100644 tools/pipelines/templates/test-template-deployment-job.yml diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index ff19587e5fef..d931f5265b95 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -185,17 +185,16 @@ extends: customBuildTags: - ES365AIMigrationTooling stages: - # - ${{ if ne(convertToJson(parameters.checks), '[]') }}: - # - template: /tools/pipelines/templates/include-policy-check.yml@self - # parameters: - # buildDirectory: '${{ parameters.buildDirectory }}' - # checks: '${{ parameters.checks }}' - # # Install all dependencies, not just the root ones - # dependencyInstallCommand: pnpm install --frozen-lockfile - - # Publish stage - # - ${{ if eq(parameters.publish, true) }}: - - template: /tools/pipelines/templates/include-publish-npm-package.yml@self + - ${{ if ne(convertToJson(parameters.checks), '[]') }}: + - template: /tools/pipelines/templates/include-policy-check.yml@self + parameters: + buildDirectory: '${{ parameters.buildDirectory }}' + checks: '${{ parameters.checks }}' + # Install all dependencies, not just the root ones + dependencyInstallCommand: pnpm install --frozen-lockfile + + # Test deployment job + - template: /tools/pipelines/templates/test-template-deployment-job.yml@self parameters: tagName: ${{ parameters.tagName }} isReleaseGroup: ${{ parameters.isReleaseGroup }} @@ -804,14 +803,14 @@ extends: buildDirectory: '${{ parameters.buildDirectory }}' testResultDirs: '${{ parameters.testResultDirs }}' - # # Publish stage - # - ${{ if eq(parameters.publish, true) }}: - # - template: /tools/pipelines/templates/include-publish-npm-package.yml@self - # parameters: - # tagName: ${{ parameters.tagName }} - # isReleaseGroup: ${{ parameters.isReleaseGroup }} - # buildDirectory: ${{ parameters.buildDirectory }} - # buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} + # Publish stage + - ${{ if eq(parameters.publish, true) }}: + - template: /tools/pipelines/templates/include-publish-npm-package.yml@self + parameters: + tagName: ${{ parameters.tagName }} + isReleaseGroup: ${{ parameters.isReleaseGroup }} + buildDirectory: ${{ parameters.buildDirectory }} + buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Capture pipeline stage results - ${{ if eq(parameters.telemetry, true) }}: diff --git a/tools/pipelines/templates/build-npm-package.yml b/tools/pipelines/templates/build-npm-package.yml index c308fe9dfc14..3c49984cb801 100644 --- a/tools/pipelines/templates/build-npm-package.yml +++ b/tools/pipelines/templates/build-npm-package.yml @@ -146,6 +146,12 @@ resources: type: git name: 1ESPipelineTemplates/M365GPT ref: refs/tags/release + pipelines: + # Listing this pipeline as a resource makes its artifacts automatically downloaded within a deployment job. + - pipeline: buildTools-resource + project: internal + source: Build - build-tools + branch: main extends: # The pipeline extends the 1ES pipeline template which will inject different SDL and compliance tasks. diff --git a/tools/pipelines/templates/test-template-deployment-job.yml b/tools/pipelines/templates/test-template-deployment-job.yml new file mode 100644 index 000000000000..c1f7d32cfbce --- /dev/null +++ b/tools/pipelines/templates/test-template-deployment-job.yml @@ -0,0 +1,94 @@ +# Copyright (c) Microsoft Corporation and contributors. All rights reserved. +# Licensed under the MIT License. + +# include-publish-npm-package-deployment + +parameters: +- name: feedKind + type: string + +- name: feedUrl + type: string + +- name: environment + type: string + +- name: pool + type: object + default: Small-eastus2 + +- name: publishFlags + type: string + default: + +- name: customEndPoint + type: string + default: + +- name: tagName + type: string + default: + +- name: buildDirectory + type: string + +- name: buildToolsVersionToInstall + type: string + default: repo + +# The path to the pnpm store. +- name: pnpmStorePath + type: string + default: $(Pipeline.Workspace)/.pnpm-store + +jobs: +- deployment: publish_${{ replace(parameters.environment, '-', '_') }} + # Depend on the build job + # dependsOn: checkoutAndUploadThisRepoAsArtifact + displayName: Publish ${{ parameters.environment }} + pool: ${{ parameters.pool }} + environment: ${{ parameters.environment }} + # This templateContext section is necessary for 1ES compliance for deployment jobs. + # see https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/releasepipelines/releaseworkflows/releasejob?tabs=combined-pipeline for more info. + templateContext: + type: releaseJob + isProduction: true + # Inputs here are automatically downloaded at the beginning of the job. + # inputs: + # - input: pipelineArtifact + # artifactName: pack + # buildType: current + # targetPath: $(Pipeline.Workspace)/pack + + workspace: + clean: all + variables: + version: $[ stageDependencies.build.build.outputs['SetVersion.version']] + isLatest: $[ stageDependencies.build.build.outputs['SetVersion.isLatest']] + strategy: + runOnce: + deploy: + steps: + - template: /tools/pipelines/templates/include-use-node-version.yml@self + + - task: Bash@3 + name: listFilesInWorkspace + displayName: List files in worksspace + inputs: + targetType: 'inline' + workingDirectory: '$(Pipeline.Workspace)' + script: | + echo "Listing files in directory: $(pwd)" + ls -la + + - task: Bash@3 + name: InstallBuildToolsFromTarball + displayName: Install Fluid Build Tools from Artifact Tarball + inputs: + targetType: 'inline' + workingDirectory: '$(Pipeline.Workspace)/buildTools-resource/pack/tarballs' + script: | + echo "Listing files in directory: $(pwd)" + ls -la + echo "attempting tarball install" + npm i -g ./*.tgz From d956161cda4986f74e790e0f2a561121f27ce7bd Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:21:26 +0000 Subject: [PATCH 56/58] removes parameters from test deployment template job --- .../templates/build-npm-client-package.yml | 5 ---- .../test-template-deployment-job.yml | 28 ------------------- 2 files changed, 33 deletions(-) diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index d931f5265b95..0e290bd11788 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -195,11 +195,6 @@ extends: # Test deployment job - template: /tools/pipelines/templates/test-template-deployment-job.yml@self - parameters: - tagName: ${{ parameters.tagName }} - isReleaseGroup: ${{ parameters.isReleaseGroup }} - buildDirectory: ${{ parameters.buildDirectory }} - buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }} # Install / Build / Test Stage - stage: build diff --git a/tools/pipelines/templates/test-template-deployment-job.yml b/tools/pipelines/templates/test-template-deployment-job.yml index c1f7d32cfbce..6bcf96f20fb0 100644 --- a/tools/pipelines/templates/test-template-deployment-job.yml +++ b/tools/pipelines/templates/test-template-deployment-job.yml @@ -4,38 +4,10 @@ # include-publish-npm-package-deployment parameters: -- name: feedKind - type: string - -- name: feedUrl - type: string - -- name: environment - type: string - - name: pool type: object default: Small-eastus2 -- name: publishFlags - type: string - default: - -- name: customEndPoint - type: string - default: - -- name: tagName - type: string - default: - -- name: buildDirectory - type: string - -- name: buildToolsVersionToInstall - type: string - default: repo - # The path to the pnpm store. - name: pnpmStorePath type: string From 1a20fcc16c5e2bc3b7fdf2098f991e8b708a4201 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:22:53 +0000 Subject: [PATCH 57/58] Removes all paramters from test deployment template except pool --- tools/pipelines/templates/test-template-deployment-job.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/pipelines/templates/test-template-deployment-job.yml b/tools/pipelines/templates/test-template-deployment-job.yml index 6bcf96f20fb0..b3b59fe9ad76 100644 --- a/tools/pipelines/templates/test-template-deployment-job.yml +++ b/tools/pipelines/templates/test-template-deployment-job.yml @@ -8,11 +8,6 @@ parameters: type: object default: Small-eastus2 -# The path to the pnpm store. -- name: pnpmStorePath - type: string - default: $(Pipeline.Workspace)/.pnpm-store - jobs: - deployment: publish_${{ replace(parameters.environment, '-', '_') }} # Depend on the build job From 21451ca4259dfc2f4e03fba17c5f28e1ff0c7cd4 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:27:13 +0000 Subject: [PATCH 58/58] adds environemnt paramter to test deployment template --- tools/pipelines/templates/test-template-deployment-job.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/pipelines/templates/test-template-deployment-job.yml b/tools/pipelines/templates/test-template-deployment-job.yml index b3b59fe9ad76..1e54024ef3c1 100644 --- a/tools/pipelines/templates/test-template-deployment-job.yml +++ b/tools/pipelines/templates/test-template-deployment-job.yml @@ -7,6 +7,9 @@ parameters: - name: pool type: object default: Small-eastus2 +- name: environment + type: string + default: test-package-build-feed jobs: - deployment: publish_${{ replace(parameters.environment, '-', '_') }}