Cumulus 3642 connect db

CHANGELOG.md

@@ -6,6 +6,31 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Replace ElasticSearch Phase 1
+- **CUMULUS-3699**
+ - Updated `collections` api endpoint to be able to support `includeStats` query string parameter
+- **CUMULUS-3641**
+ - Updated `collections` api endpoint to query postgres instead of elasticsearch except if `includeStats` is in the query parameters
+- **CUMULUS-3695**
+ - Updated `granule` list api endpoint and BaseSearch class to handle sort fields
+- **CUMULUS-3688**
+ - Updated `stats` api endpoint to query postgres instead of elasticsearch
+- **CUMULUS-3689**
+ - Updated `stats/aggregate` api endpoint to query postgres instead of elasticsearch
+ - Created a new StatsSearch class for querying postgres with the stats endpoint
+- **CUMULUS-3692**
+  - Added `@cumulus/db/src/search` `BaseSearch` and `GranuleSearch` classes to
+    support basic queries for granules
+  - Updated granules List endpoint to query postgres for basic queries
+- **CUMULUS-3693**
+  - Added functionality to `@cumulus/db/src/search` to support range queries
+- **CUMULUS-3694**
+  - Added functionality to `@cumulus/db/src/search` to support term queries
+  - Updated `BaseSearch` and `GranuleSearch` classes to support term queries for granules
+  - Updated granules List endpoint to search postgres
+- **CUMULUS-3696**
+  - Added functionality to `@cumulus/db/src/search` to support terms, `not` and `exists` queries
+
 ### Migration Notes
 
 #### CUMULUS-3433 Update to node.js v20

bamboo/bootstrap-tf-deployment.sh

@@ -96,7 +96,8 @@ echo "Deploying Cumulus example to $DEPLOYMENT"
   -var "token_secret=$TOKEN_SECRET" \
   -var "permissions_boundary_arn=arn:aws:iam::$AWS_ACCOUNT_ID:policy/$ROLE_BOUNDARY" \
   -var "pdr_node_name_provider_bucket=$PDR_NODE_NAME_PROVIDER_BUCKET" \
-  -var "rds_admin_access_secret_arn=$RDS_ADMIN_ACCESS_SECRET_ARN" \
+  -var "rds_security_group=$OTHER_DB_RDS_SECURITY_GROUP"\
+  -var "rds_admin_access_secret_arn=$OTHER_DB_RDS_ADMIN_ACCESS_SECRET_ARN" \
   -var "orca_db_user_password=$ORCA_DATABASE_USER_PASSWORD" \
   -var "orca_s3_access_key=$AWS_ACCESS_KEY_ID" \
   -var "orca_s3_secret_key=$AWS_SECRET_ACCESS_KEY" \

bamboo/deploy-dev-integration-test-stack.sh

@@ -4,7 +4,7 @@ set -ex
 . ./bamboo/use-working-directory.sh
 . ./bamboo/set-bamboo-env-variables.sh
 . ./bamboo/abort-if-not-pr-or-redeployment.sh
-. ./bamboo/abort-if-skip-integration-tests.sh
+# . ./bamboo/abort-if-skip-integration-tests.sh
 
 if [[ $USE_TERRAFORM_ZIPS == true ]]; then
   ## If this flag is set, we want to use the output of the 'publish' stage

bamboo/set-bamboo-env-variables.sh

@@ -146,6 +146,8 @@ if [[ $bamboo_NGAP_ENV = "SIT" ]]; then
   export PDR_NODE_NAME_PROVIDER_BUCKET=$bamboo_SIT_PDR_NODE_NAME_PROVIDER_BUCKET
   export ORCA_DATABASE_USER_PASSWORD=$bamboo_SECRET_SIT_ORCA_DATABASE_USER_PASSWORD
   DEPLOYMENT=$bamboo_SIT_DEPLOYMENT
+  export OTHER_DB_RDS_SECURITY_GROUP=$bamboo_SECRET_SIT_OTHER_DB_RDS_SECURITY_GROUP
+  export OTHER_DB_RDS_ADMIN_ACCESS_SECRET_ARN=$bamboo_SECRET_SIT_OTHER_DB_RDS_ADMIN_ACCESS_SECRET_ARN
 fi
 
 ## Run detect-pr script and set flag to true/false

example/config.yml

@@ -8,6 +8,11 @@ cumulus-sit:
   apiUsername: jasmine
   pdrNodeNameProviderBucket: cumulus-sit-pdr-node-name-provider
 
+cumulus-es:
+  bucket: cumulus-sit-internal
+  apiUsername: jasmine
+  pdrNodeNameProviderBucket: cumulus-sit-pdr-node-name-provider
+
 mvd-tf:
   bucket: mvd-internal
 

example/cumulus-tf/main.tf

@@ -38,8 +38,8 @@ locals {
   elasticsearch_security_group_id = lookup(data.terraform_remote_state.data_persistence.outputs, "elasticsearch_security_group_id", "")
   protected_bucket_names          = [for k, v in var.buckets : v.name if v.type == "protected"]
   public_bucket_names             = [for k, v in var.buckets : v.name if v.type == "public"]
-  rds_security_group              = lookup(data.terraform_remote_state.data_persistence.outputs, "rds_security_group", "")
-  rds_credentials_secret_arn      = lookup(data.terraform_remote_state.data_persistence.outputs, "database_credentials_secret_arn", "")
+  rds_security_group              = var.rds_security_group
+  rds_credentials_secret_arn      = var.rds_admin_access_secret_arn
 
   vpc_id     = var.vpc_id != null ? var.vpc_id : data.aws_vpc.application_vpc[0].id
   subnet_ids = length(var.lambda_subnet_ids) > 0 ? var.lambda_subnet_ids : data.aws_subnets.subnet_ids[0].ids

example/cumulus-tf/variables.tf

@@ -347,6 +347,10 @@ variable "rds_admin_access_secret_arn" {
   type        = string
 }
 
+variable "rds_security_group" {
+  type = string
+}
+
 variable "async_operation_image_version" {
   description = "docker image version to use for Cumulus async operations tasks"
   type = string

example/data-persistence-tf/main.tf

@@ -36,7 +36,7 @@ module "provision_database" {
   permissions_boundary_arn               = var.permissions_boundary_arn
   rds_user_password                      = var.rds_user_password == "" ? random_string.db_pass.result : var.rds_user_password
   rds_connection_timing_configuration    = var.rds_connection_timing_configuration
-  dbRecreation                           = true
+  dbRecreation                           = false
   lambda_timeouts                        = var.lambda_timeouts
   lambda_memory_sizes                    = var.lambda_memory_sizes
 }

example/deployments/cumulus/cumulus-es.tfvars

@@ -0,0 +1,4 @@
+prefix = "cumulus-es"
+archive_api_port = 8000
+key_name = "lp"
+cmr_oauth_provider = "launchpad"

example/deployments/data-persistence/cumulus-es.tfvars

@@ -0,0 +1 @@
+prefix = "cumulus-es"

example/deployments/db-migration/cumulus-es.tfvars

@@ -0,0 +1 @@
+prefix = "cumulus-es"

example/spec/helpers/granuleUtils.js

@@ -234,6 +234,7 @@ const waitForGranuleRecordUpdatedInList = async (stackName, granule, additionalQ
       'beginningDateTime',
       'endingDateTime',
       'error',
+      'execution', // TODO remove after CUMULUS-3698
       'files', // TODO -2714 this should be removed
       'lastUpdateDateTime',
       'productionDateTime',

example/spec/parallel/testAPI/granuleSpec.js

@@ -183,7 +183,8 @@ describe('The Granules API', () => {
       });
 
       const searchedGranule = JSON.parse(searchResults.body).results[0];
-      expect(searchedGranule).toEqual(jasmine.objectContaining(randomGranuleRecord));
+      // TODO CUMULUS-3698 includes files
+      expect(searchedGranule).toEqual(jasmine.objectContaining(omit(randomGranuleRecord, 'files')));
     });
 
     it('can modify the granule via API.', async () => {

packages/api/endpoints/collections.js

@@ -1,3 +1,5 @@
+//@ts-check
+
 'use strict';
 
 const router = require('express-promise-router')();
@@ -16,6 +18,7 @@ const {
   isCollisionError,
   translateApiCollectionToPostgresCollection,
   translatePostgresCollectionToApiCollection,
+  CollectionSearch,
 } = require('@cumulus/db');
 const CollectionConfigStore = require('@cumulus/collection-config-store');
 const { getEsClient, Search } = require('@cumulus/es-client/search');
@@ -43,14 +46,12 @@ const log = new Logger({ sender: '@cumulus/api/collections' });
  * @returns {Promise<Object>} the promise of express response object
  */
 async function list(req, res) {
-  const { getMMT, includeStats, ...queryStringParameters } = req.query;
-  const collection = new Collection(
-    { queryStringParameters },
-    undefined,
-    process.env.ES_INDEX,
-    includeStats === 'true'
+  log.trace(`list query ${JSON.stringify(req.query)}`);
+  const { getMMT, ...queryStringParameters } = req.query;
+  const dbSearch = new CollectionSearch(
+    { queryStringParameters }
   );
-  let result = await collection.query();
+  let result = await dbSearch.query();
   if (getMMT === 'true') {
     result = await insertMMTLinks(result);
   }

packages/api/endpoints/granules.js

@@ -12,6 +12,7 @@ const { v4: uuidv4 } = require('uuid');
 const Logger = require('@cumulus/logger');
 const { deconstructCollectionId } = require('@cumulus/message/Collections');
 const { RecordDoesNotExist } = require('@cumulus/errors');
+const { GranuleSearch } = require('@cumulus/db');
 
 const {
   CollectionPgModel,
@@ -31,7 +32,6 @@ const {
   recordNotFoundString,
   multipleRecordFoundString,
 } = require('@cumulus/es-client/search');
-const ESSearchAfter = require('@cumulus/es-client/esSearchAfter');
 
 const { deleteGranuleAndFiles } = require('../src/lib/granule-delete');
 const { zodParser } = require('../src/zod-utils');
@@ -101,19 +101,12 @@ function _createNewGranuleDateValue() {
  * @returns {Promise<Object>} the promise of express response object
  */
 async function list(req, res) {
+  log.trace(`list query ${JSON.stringify(req.query)}`);
   const { getRecoveryStatus, ...queryStringParameters } = req.query;
 
-  let es;
-  if (queryStringParameters.searchContext) {
-    es = new ESSearchAfter(
-      { queryStringParameters },
-      'granule',
-      process.env.ES_INDEX
-    );
-  } else {
-    es = new Search({ queryStringParameters }, 'granule', process.env.ES_INDEX);
-  }
-  const result = await es.query();
+  const dbSearch = new GranuleSearch({ queryStringParameters });
+  const result = await dbSearch.query();
+
   if (getRecoveryStatus === 'true') {
     return res.send(await addOrcaRecoveryStatus(result));
   }

packages/api/endpoints/stats.js

@@ -2,7 +2,8 @@
 
 const router = require('express-promise-router')();
 const get = require('lodash/get');
-const Stats = require('@cumulus/es-client/stats');
+const { StatsSearch } = require('@cumulus/db');
+const omit = require('lodash/omit');
 
 /**
  * Map requested stats types to supported types
@@ -34,17 +35,8 @@ function getType(req) {
  * @returns {Promise<Object>} the promise of express response object
  */
 async function summary(req, res) {
-  const params = req.query;
-
-  params.timestamp__from = Number.parseInt(get(
-    params,
-    'timestamp__from',
-    0
-  ), 10);
-  params.timestamp__to = Number.parseInt(get(params, 'timestamp__to', Date.now()), 10);
-
-  const stats = new Stats({ queryStringParameters: params }, undefined, process.env.ES_INDEX);
-  const r = await stats.query();
+  const stats = new StatsSearch({ queryStringParameters: req.query }, 'granule');
+  const r = await stats.summary();
   return res.send(r);
 }
 
@@ -56,13 +48,12 @@ async function summary(req, res) {
  * @returns {Promise<Object>} the promise of express response object
  */
 async function aggregate(req, res) {
-  const type = getType(req);
-
-  const stats = new Stats({
-    queryStringParameters: req.query,
-  }, type, process.env.ES_INDEX);
-  const r = await stats.count();
-  return res.send(r);
+  if (getType(req)) {
+    const stats = new StatsSearch({ queryStringParameters: omit(req.query, 'type') }, getType(req));
+    const r = await stats.aggregate();
+    return res.send(r);
+  }
+  return res.boom.badRequest('Type must be included in Stats Aggregate query string parameters');
 }
 
 router.get('/aggregate/:type?', aggregate);

packages/api/tests/app/test-launchpadAuth.js

@@ -10,7 +10,7 @@ const { createBucket, putJsonS3Object } = require('@cumulus/aws-client/S3');
 const launchpad = require('@cumulus/launchpad-auth');
 const { randomId } = require('@cumulus/common/test-utils');
 
-const EsCollection = require('@cumulus/es-client/collections');
+const { CollectionSearch } = require('@cumulus/db');
 const models = require('../../models');
 const { createJwtToken } = require('../../lib/token');
 const { fakeAccessTokenFactory } = require('../../lib/testUtils');
@@ -72,7 +72,7 @@ test.after.always(async () => {
 
 test.serial('API request with a valid Launchpad token stores the access token', async (t) => {
   const stub = sinon.stub(launchpad, 'validateLaunchpadToken').returns(validateTokenResponse);
-  const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]);
+  const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]);
 
   try {
     await request(app)
@@ -113,7 +113,7 @@ test.serial('API request with an invalid Launchpad token returns a 403 unauthori
 
 test.serial('API request with a stored non-expired Launchpad token record returns a successful response', async (t) => {
   let stub = sinon.stub(launchpad, 'validateLaunchpadToken').resolves(validateTokenResponse);
-  const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]);
+  const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]);
 
   try {
     await request(app)
@@ -143,7 +143,7 @@ test.serial('API request with a stored non-expired Launchpad token record return
 });
 
 test.serial('API request with an expired Launchpad token returns a 401 response', async (t) => {
-  const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]);
+  const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]);
 
   try {
     await accessTokenModel.create({