CUMULUS-3779: Excludes aws-sdk v3 from webpack packages (#3689)

* Excludes aws-sdk v3 from webpack packages

* Updates CL; Excludes aws-sdk v3 from new lambdas

* Pushes new async_operations docker image. Bumps default tag

* Updates entrypoint for async operations docker image

* WIP: Testing CI against latest cumulus-ecs-task docker image

* WIP: Bumps async_operations cumulus dependencies to v18.3.0

* WIP: Adds CUMULUS-3779 to CHANGELOG with migration notes

* WIP: Bumps default cumulus-ecs-task version to 2.1.0

CHANGELOG.md

@@ -62,6 +62,11 @@ output or status of your request. If you want to directly observe the progress
 of the migration as it runs, you can view the CloudWatch logs for your async
 operations (e.g. `PREFIX-AsyncOperationEcsLogs`).
 
+#### CUMULUS-3779 async_operations Docker image version upgrade
+
+The `async_operations` Docker image has been updated to support Node v20 and `aws-sdk` v3. Users will need to bump
+the version tag of `async_operations` to at least 52 if using the Docker image.
+
 ### Breaking Changes
 
 - **CUMULUS-3618**
@@ -103,6 +108,9 @@ operations (e.g. `PREFIX-AsyncOperationEcsLogs`).
   - Changed granules table unique constraint to granules_collection_cumulus_id_granule_id_unique
   - Added indexes granules_granule_id_index and granules_provider_collection_cumulus_id_granule_id_index
     to granules table
+- **CUMULUS-3779**
+  - Updates async_operations Docker image to Node v20 and bumps its cumulus dependencies to v18.3.0 to
+    support `aws-sdk` v3 changes.
 
 ### Added
 - **CUMULUS-3742**
@@ -216,6 +224,10 @@ operations (e.g. `PREFIX-AsyncOperationEcsLogs`).
     credentialing timeout in long-running ECS jobs.
 - **CUMULUS-3323**
   - Minor edits to errant integration test titles (dyanmo->postgres)
+- **AWS-SDK v3 Exclusion (v18.3.0 fix)***
+  - Excludes aws-sdk v3 from packages to reduce overall package size. With the requirement of Node v20
+    packaging the aws-sdk v3 with our code is no longer necessary and prevented some packages from being
+    published to npm.
 
 ## [v18.2.2] 2024-06-4
 

example/cumulus-tf/variables.tf

@@ -350,7 +350,7 @@ variable "rds_admin_access_secret_arn" {
 variable "async_operation_image_version" {
   description = "docker image version to use for Cumulus async operations tasks"
   type = string
-  default = "49"
+  default = "52"
 }
 
 variable "cumulus_process_activity_version" {
@@ -362,7 +362,7 @@ variable "cumulus_process_activity_version" {
 variable "ecs_task_image_version" {
   description = "docker image version to use for Cumulus hello world task"
     type = string
-    default = "1.9.0"
+    default = "2.1.0"
 }
 
 variable "cumulus_test_ingest_image_version" {

lambdas/db-migration/webpack.config.js

@@ -80,5 +80,8 @@ module.exports = {
   node: {
     __dirname: false
   },
-  target: 'node'
+  target: 'node',
+  externals: [
+    /@aws-sdk\//
+  ]
 };

lambdas/db-provision-user-database/webpack.config.js

@@ -46,5 +46,8 @@ module.exports = {
       },
     ],
   },
-  target: 'node'
+  target: 'node',
+  externals: [
+    /@aws-sdk\//
+  ]
 };

lambdas/dla-migration/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist', 'webpack'),
     filename: 'index.js'
   },
-  externals: ['aws-sdk'],
+  externals: [/@aws-sdk\//],
   target: 'node',
   devtool: 'eval-cheap-module-source-map',
   optimization: {

lambdas/migration-helper-async-operation/webpack.config.js

@@ -46,5 +46,8 @@ module.exports = {
       },
     ],
   },
-  target: 'node'
+  target: 'node',
+  externals: [
+    /@aws-sdk\//
+  ]
 };

lambdas/sqs-message-remover/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

packages/api/ecs/async-operation/Dockerfile

@@ -1,18 +1,21 @@
-FROM node:20.12.2-buster
+FROM amazon/aws-lambda-nodejs:20
 
 USER root
-RUN sed -i -e '/jessie-updates/d' /etc/apt/sources.list
-RUN apt-get update && apt-get install -y unzip
+RUN dnf install -y unzip shadow-utils
 
-RUN groupadd -r task -g 433
-RUN useradd -u 431 -r -g task -m -s /sbin/nologin -c "Docker image user" task
+RUN /usr/sbin/groupadd -r task -g 433
+RUN /usr/sbin/useradd -u 431 -r -g task -m -s /sbin/nologin -c "Docker image user" task
 
 USER task
 WORKDIR /home/task
 
 COPY package.json /home/task/
 RUN npm install
 
+RUN cp -a /var/runtime/node_modules/. /home/task/node_modules/
+
 COPY index.js /home/task/
 
 CMD [ "node", "--harmony", "index.js" ]
+
+ENTRYPOINT [ ]

packages/api/ecs/async-operation/package.json

@@ -22,10 +22,10 @@
     "coverage": "python ../../../../scripts/coverage_handler/coverage.py"
   },
   "dependencies": {
-    "@cumulus/aws-client": "18.2.0",
-    "@cumulus/db": "18.2.0",
-    "@cumulus/es-client": "18.2.0",
-    "@cumulus/logger": "18.2.0",
+    "@cumulus/aws-client": "18.3.0",
+    "@cumulus/db": "18.3.0",
+    "@cumulus/es-client": "18.3.0",
+    "@cumulus/logger": "18.3.0",
     "@aws-sdk/client-lambda": "^3.529.1",
     "crypto-random-string": "^3.2.0",
     "got": "^11.8.5",

packages/api/webpack.config.js

@@ -80,6 +80,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     { formidable: 'url' },
     { fsevents: "require('fsevents')" }

packages/integration-tests/webpack.config.js

@@ -31,6 +31,7 @@ module.exports = {
     },
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     { formidable: 'url' },
   ],

packages/s3-credentials-endpoint/webpack.config.js

@@ -32,6 +32,7 @@ module.exports = {
     },
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     { formidable: 'url' },
   ],

packages/tea-map-cache/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist'),
     filename: 'index.js'
   },
-  externals: [],
+  externals: [/@aws-sdk\//],
   target: 'node',
   devtool: 'eval-cheap-module-source-map',
   optimization: {

tasks/add-missing-file-checksums/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist', 'webpack'),
     filename: 'index.js'
   },
-  externals: [],
+  externals: [/@aws-sdk\//],
   target: 'node',
   devtool: 'eval-cheap-module-source-map',
   optimization: {

tasks/discover-granules/webpack.config.js

@@ -27,6 +27,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/discover-pdrs/webpack.config.js

@@ -27,6 +27,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/files-to-granules/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/hello-world/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/hyrax-metadata-updates/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/lzards-backup/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist', 'webpack'),
     filename: 'index.js'
   },
-  externals: [],
+  externals: [/@aws-sdk\//],
   target: 'node',
   devtool: 'eval-cheap-module-source-map',
   optimization: {

tasks/move-granules/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'},
     // See https://github.com/knex/knex/issues/1128 re: webpack configuration

tasks/orca-copy-to-archive-adapter/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist', 'webpack'),
     filename: 'index.js'
   },
-  externals: [],
+  externals: [/@aws-sdk\//],
   target: 'node',
   devtool: 'eval-cheap-module-source-map',
   optimization: {

tasks/orca-recovery-adapter/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist', 'webpack'),
     filename: 'index.js'
   },
-  externals: [],
+  externals: [/@aws-sdk\//],
   target: 'node',
   devtool: 'eval-cheap-module-source-map',
   optimization: {

tasks/parse-pdr/webpack.config.js

@@ -27,6 +27,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/pdr-status-check/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/post-to-cmr/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/queue-granules/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist', 'webpack'),
     filename: 'index.js'
   },
-  externals: [],
+  externals: [/@aws-sdk\//],
   target: 'node',
   devtool: 'eval-cheap-module-source-map',
   optimization: {

tasks/queue-pdrs/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/queue-workflow/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/send-pan/webpack.config.js

@@ -31,7 +31,7 @@ module.exports = {
     path: path.resolve(__dirname, 'dist', 'webpack'),
     filename: 'index.js'
   },
-  externals: [],
+  externals: [/@aws-sdk\//],
   target: 'node',
   devtool: 'eval-cheap-module-source-map',
   optimization: {

tasks/sf-sqs-report/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     { formidable: 'url' }
   ],

tasks/sync-granule/webpack.config.js

@@ -27,6 +27,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'},
     // See https://github.com/knex/knex/issues/1128 re: webpack configuration

tasks/test-processing/webpack.config.js

@@ -31,6 +31,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     { formidable: 'url' },
   ],

tasks/update-cmr-access-constraints/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tasks/update-granules-cmr-metadata-file-links/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],

tf-modules/internal/cumulus-test-cleanup/webpack.config.js

@@ -16,6 +16,7 @@ module.exports = {
     }
   },
   externals: [
+    /@aws-sdk\//,
     'electron',
     {'formidable': 'url'}
   ],