From 8c7f28c9e31925e80c4d291e781ab6ca8b656cae Mon Sep 17 00:00:00 2001 From: Sara Reynolds Date: Mon, 26 Aug 2019 16:18:07 -0700 Subject: [PATCH] Trusted Advisor Integration V1 --- deployment/build_packages.sh | 2 +- deployment/cf-templates/ddb.json | 64 +++- .../identification-crossaccount-role.json | 8 + .../cf-templates/identification-role.json | 8 + deployment/cf-templates/identification.json | 67 ++++- deployment/configs/config.json | 53 +++- .../playbook24_trusted_advisor_checks.md | 274 ++++++++++++++++++ docs/pages/playbook25_adding_a_rule.md | 97 +++++++ .../README.md | 193 ++++++++++++ .../describe_ta_checks.py | 75 +++++ .../initiate_to_desc_ta_checks.py | 97 +++++++ hammer/library/aws/ta.py | 105 +++++++ hammer/library/config.py | 14 + hammer/library/ddb_issues.py | 14 +- 14 files changed, 1054 insertions(+), 17 deletions(-) create mode 100644 docs/pages/playbook24_trusted_advisor_checks.md create mode 100644 docs/pages/playbook25_adding_a_rule.md create mode 100644 hammer/identification/lambdas/trusted-advisor-checks-identification/README.md create mode 100755 hammer/identification/lambdas/trusted-advisor-checks-identification/describe_ta_checks.py create mode 100755 hammer/identification/lambdas/trusted-advisor-checks-identification/initiate_to_desc_ta_checks.py create mode 100755 hammer/library/aws/ta.py diff --git a/deployment/build_packages.sh b/deployment/build_packages.sh index d0c53c5a..5f1690ec 100755 --- a/deployment/build_packages.sh +++ b/deployment/build_packages.sh @@ -23,7 +23,7 @@ SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )" PACKAGES_DIR="${SCRIPT_PATH}/packages/" LIBRARY="${SCRIPT_PATH}/../hammer/library" -LAMBDAS="ami-info logs-forwarder ddb-tables-backup sg-issues-identification s3-acl-issues-identification s3-policy-issues-identification iam-keyrotation-issues-identification iam-user-inactive-keys-identification cloudtrails-issues-identification ebs-unencrypted-volume-identification ebs-public-snapshots-identification rds-public-snapshots-identification sqs-public-policy-identification s3-unencrypted-bucket-issues-identification rds-unencrypted-instance-identification ami-public-access-issues-identification api ecs-privileged-access-issues-identification ecs-logging-issues-identification ecs-external-image-source-issues-identification redshift-audit-logging-issues-identification redshift-unencrypted-cluster-identification redshift-cluster-public-access-identification elasticsearch-domain-logging-issues-identification elasticsearch-unencrypted-domain-identification elasticsearch-public-access-domain-identification" +LAMBDAS="ami-info logs-forwarder ddb-tables-backup sg-issues-identification s3-acl-issues-identification s3-policy-issues-identification iam-keyrotation-issues-identification iam-user-inactive-keys-identification cloudtrails-issues-identification ebs-unencrypted-volume-identification ebs-public-snapshots-identification rds-public-snapshots-identification sqs-public-policy-identification s3-unencrypted-bucket-issues-identification rds-unencrypted-instance-identification ami-public-access-issues-identification api ecs-privileged-access-issues-identification ecs-logging-issues-identification ecs-external-image-source-issues-identification redshift-audit-logging-issues-identification redshift-unencrypted-cluster-identification redshift-cluster-public-access-identification elasticsearch-domain-logging-issues-identification elasticsearch-unencrypted-domain-identification elasticsearch-public-access-domain-identification trusted-advisor-checks-identification" pushd "${SCRIPT_PATH}" > /dev/null pushd ../hammer/identification/lambdas > /dev/null diff --git a/deployment/cf-templates/ddb.json b/deployment/cf-templates/ddb.json index 82a39012..cbc8fed5 100755 --- a/deployment/cf-templates/ddb.json +++ b/deployment/cf-templates/ddb.json @@ -512,7 +512,7 @@ }, "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "ecs-privileged-access" ] ]} } - }, + }, "DynamoDBECSLogging": { "Type": "AWS::DynamoDB::Table", "DeletionPolicy": "Retain", @@ -734,7 +734,7 @@ }, "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "es-unencrypted-domain" ] ]} } - }, + }, "DynamoDBESPublicAccessRequests": { "Type": "AWS::DynamoDB::Table", "DependsOn": ["DynamoDBCredentials"], @@ -765,6 +765,66 @@ }, "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "es-public-access-domain" ] ]} } + }, + "DynamoDBTrustedAdvisorEC2LowUtil": { + "Type": "AWS::DynamoDB::Table", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "BillingMode": "PAY_PER_REQUEST", + "SSESpecification": { + "SSEEnabled": true + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "trusted-advisor-cost-optimizing-low-utilization-ec2-instances" ] ]} + } + }, + "DynamoDBTrustedAdvisorFaultToleranceEBS": { + "Type": "AWS::DynamoDB::Table", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "BillingMode": "PAY_PER_REQUEST", + "SSESpecification": { + "SSEEnabled": true + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "trusted-advisor-fault-tolerance-ebs-snapshots" ] ]} + } } } } diff --git a/deployment/cf-templates/identification-crossaccount-role.json b/deployment/cf-templates/identification-crossaccount-role.json index bb1ae517..d38f74e4 100755 --- a/deployment/cf-templates/identification-crossaccount-role.json +++ b/deployment/cf-templates/identification-crossaccount-role.json @@ -160,6 +160,14 @@ "es:ListTags" ], "Resource": "*" + }, + { + "Sid": "HammerSupportPolicy", + "Effect": "Allow", + "Action": [ + "support:*" + ], + "Resource": "*" } ] } diff --git a/deployment/cf-templates/identification-role.json b/deployment/cf-templates/identification-role.json index b2ccfb0f..20918dd0 100755 --- a/deployment/cf-templates/identification-role.json +++ b/deployment/cf-templates/identification-role.json @@ -217,6 +217,14 @@ "Resource": { "Fn::Join": ["", ["arn:aws:iam::*:role/", { "Ref": "ResourcesPrefix" }, {"Ref": "IdentificationCrossAccountIAMRole"}]] } + }, + { + "Sid": "HammerSupportPolicy", + "Effect": "Allow", + "Action": [ + "support:*" + ], + "Resource": "*" } ] } diff --git a/deployment/cf-templates/identification.json b/deployment/cf-templates/identification.json index 82caa983..c7cb28d4 100755 --- a/deployment/cf-templates/identification.json +++ b/deployment/cf-templates/identification.json @@ -36,7 +36,8 @@ "SourceIdentificationECSExternalImageSource", "SourceIdentificationElasticSearchLogging", "SourceIdentificationElasticSearchEncryption", - "SourceIdentificationElasticSearchPublicAccess" + "SourceIdentificationElasticSearchPublicAccess", + "SourceIdentificationTAChecks" ] }, { @@ -127,7 +128,10 @@ "dafault": "Relative path to Unencrypted Elasticsearch domain sources" }, "SourceIdentificationElasticSearchPublicAccess":{ - "dafault": "Relative path to Unencrypted Elasticsearch domain public access sources" + "dafault": "Relative path to Unencrypted Elasticsearch domain public access sources" + }, + "SourceIdentificationTAChecks": { + "default": "Relative path to trusted advisor check sources" } } } @@ -247,7 +251,7 @@ }, "SourceIdentificationECSExternalImageSource": { "Type": "String", - "Default": "ecs-image-source-issues-identification.zip" + "Default": "ecs-external-image-source-issues-identification.zip" }, "SourceIdentificationElasticSearchLogging": { "Type": "String", @@ -260,7 +264,11 @@ "SourceIdentificationElasticSearchPublicAccess": { "Type": "String", "Default": "elasticsearch-public-access-domain-identification.zip" - } + }, + "SourceIdentificationTAChecks": { + "Type": "String", + "Default": "trusted-advisor-checks-identification.zip" + } }, "Conditions": { "LambdaSubnetsEmpty": { @@ -473,6 +481,12 @@ "SNSTopicNameESPublicAccess": { "value": "describe-es-public-access-lambda" }, + "SNSDisplayNameTAChecks": { + "value": "describe-ta-checks-sns" + }, + "SNSTopicNameTAChecks": { + "value": "describe-ta-checks-lambda" + }, "LogsForwarderLambdaFunctionName": { "value": "logs-forwarder" }, @@ -544,6 +558,9 @@ }, "ESPublicAccessLambdaFunctionName": { "value": "elasticsearch-public-access" + }, + "TAChecksLambdaFunctionName": { + "value": "ta-checks" } } }, @@ -1592,7 +1609,47 @@ { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameESPublicAccess", "value"] } ] ]}, "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameESPublicAccess", "value"] } ] + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameESPublicAccess", "value"] } ] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateTAChecks": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + { "Ref": "ResourcesPrefix" }, + { "Ref": "IdentificationIAMRole" } + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "40 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationTAChecks" }, + "InitiateLambdaDescription": "Lambda function for initiate trusted advisor's checks.", + "EvaluateLambdaDescription": "Lambda function to describe trusted advisor's checks.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "TAChecksLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "TAChecksLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_ta_checks.lambda_handler", + "EvaluateLambdaHandler": "describe_ta_checks.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate Trusted Advisor Checks", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationTAChecks"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameTAChecks", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameTAChecks", "value"] } ] ]}, "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} } diff --git a/deployment/configs/config.json b/deployment/configs/config.json index 3f72b475..0783f959 100755 --- a/deployment/configs/config.json +++ b/deployment/configs/config.json @@ -10,7 +10,7 @@ "text_field_character_limit": 32767 }, "slack": { - "enabled": true, + "enabled": false, "notify_default_owner": true, "channels": { "#hammer-slave1": ["ERROR|WARNING|ALARM|Task timed out after|Access denied"] @@ -168,7 +168,7 @@ "ddb.table_name": "hammer-redshift-logging", "topic_name": "hammer-describe-redshift-logging-lambda", "reporting": true - }, + }, "redshift_public_access": { "enabled": true, "ddb.table_name": "hammer-redshift-public-access", @@ -203,7 +203,7 @@ "ddb.table_name": "hammer-ecs-external-image-source", "reporting": true, "safe_image_sources": ["amazonaws", "artifactory"] - }, + }, "es_domain_logging": { "enabled": true, "ddb.table_name": "hammer-es-domain-logging", @@ -217,7 +217,7 @@ "ddb.table_name": "hammer-es-unencrypted-domain", "topic_name": "hammer-describe-es-encryption-lambda", "reporting": true - }, + }, "es_public_access_domain": { "enabled": true, "ddb.table_name": "hammer-es-public-access-domain", @@ -225,5 +225,50 @@ "reporting": true, "remediation": false, "remediation_retention_period": 21 + }, + "trusted_advisor_recommendations": { + "enabled": true, + "refreshtimeoutinminutes": 8, + "checks": [ + { + "category": "cost_optimizing", + "checkname": "Low Utilization Amazon EC2 Instances", + "name": "trusted_advisor_recommendations_low_ec2_utilization", + "accounts": [ + "123456789012", + "098765432109" + ], + "filters": [ + { + "attribute": "Estimated Monthly Savings", + "operator": "gt", + "value": "500" + }, + { + "attribute": "Number of Days Low Utilization", + "operator": "eq", + "value": "14" + } + ], + "ddb.table_name": "trusted-advisor-cost-optimizing-low-utilization-ec2-instances" + }, + { + "category": "fault_tolerance", + "checkname": "Amazon EBS Snapshots", + "name": "trusted_advisor_recommendations_ebs_snapshots", + "accounts": [ + "123456789012", + "098765432109" + ], + "filters": [ + { + "attribute": "Status", + "operator": "eq", + "value": "Red" + } + ], + "ddb.table_name": "trusted-advisor-fault-tolerance-ebs-snapshots" + } + ] } } diff --git a/docs/pages/playbook24_trusted_advisor_checks.md b/docs/pages/playbook24_trusted_advisor_checks.md new file mode 100644 index 00000000..f25f897b --- /dev/null +++ b/docs/pages/playbook24_trusted_advisor_checks.md @@ -0,0 +1,274 @@ +--- +title: Trusted Advisor Checks +keywords: playbook24 +sidebar: mydoc_sidebar +permalink: playbook24_trusted_advisor_checks.html +--- + +# Playbook 24: Trusted Advisor Checks + +## Introduction +Enabling Trusted Advisor checks with Hammer allows you to add all rules and checks from AWS’s Trusted Advisor with Dow Jones’ Hammer Continuous Monitoring Service. + +File Structure: + +``` + +├── trusted-advisor-checks-identification <-- Parent folder for Trusted Advisor Checks +│ ├── README.md <-- This Instructions file +│   ├── initiate-ta-checks.py <-- Initiate Lambda function +│   ├── describe-ta-checks.py <-- Describe Lambda function + +``` + +## 1. Issue Identification + +Issues are flagged based on Trusted Advisor Checks. Please note, because these checks rely on the Trusted Advisor API and Support Access, only accounts that have some level of AWS Support can utilize this Hammer rule. + +When Dow Jones Hammer detects an issue through Trusted Advisor, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/trusted-advisor-checks-identification/initiate_to_desc_ta_checks.py`| +|Identification|`hammer/identification/lambdas/trusted-advisor-checks-identification/describe_ta_checks.py`| + +## 2. Issue Reporting and Remediation + +Issue Reporting/Remediation is currently not enabled with Trusted Advisor checks, but could be a great feature add! + + +## 3. Setup Instructions For This Issue + +In this section, we'll go through the configuration file, and all edits and additions you can make. + +### 3.1. The config.json File +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/configs/config.json`. +Review the following parameters in the **trusted-advisor-recommendations** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`refreshtimeoutinminutes` |Amount of time before the lambda initiate function is given to refresh trusted advisor checks| `8` | +|`checks` |A list of check objects with parameters that describe the specific Trusted Advisor checks. |`[]`| + +Each **check object** within the list of checks is comprised of the following: +|Parameter Name |Description | Example| +|------------------------------|---------------------------------------|:------------:| +|`category` |The Trusted Advisor specified category. |`cost_optimizing`| +|`checkname` |The Trusted Advisor specified checkname.| `Low Utilization Amazon EC2 Instances` | +|`name` |User given name. |`trusted_advisor_recommendations_low_ec2_utilization`| +|`accounts` |List of account IDs you'd like to enable the checks on. |`["12345678901","098765432109]"`| +|`filters` |A list of settings to filter the returned checks by. Default value is an empty list. |`[]`| +|`ddb.table_name` |The name of the dynamo table to hold all returned warnings from Trsuted Advisor. It is check specific and must exist in the environment! | `cost-optimizing-low-utilization-ec2-instances` + +Below is a larger example of a config section! + +``` +{ + "trusted_advisor_recommendations": { + "enabled": true, + "refreshtimeoutinminutes": 8, + "checks": [ + { + "category": "cost_optimizing", + "checkname": "Low Utilization Amazon EC2 Instances", + "name": "trusted_advisor_recommendations_low_ec2_utilization", + "accounts": [ + "Account ID 1", + "Account ID 2" + ], + "filters": [ + { + "attribute": "Estimated Monthly Savings", + "operator": "gt", + "value": "500" + }, + { + "attribute": "Number of Days Low Utilization", + "operator": "eq", + "value": "14", + } + ], + "ddb.table_name": "hammer-trusted-advisor-cost-optimizing-low-utilization-ec2-instances" + }, + { + "category": "fault_tolerance", + "checkname": "Amazon EBS Snapshots", + "accounts": [ + "Account 1", + "Account 2" + ], + "filters": [ + { + "attribute": "status", + "operator": "eq", + "value": "critical" + } + ], + "ddb.table_name": "hammer-trusted-advisor-fault-tolerance-ebs-snapshots" + } + ] + } +} +``` +### 3.2. How to configure Trusted Advisor checks + +1. Turn on Trusted Advisor checks by changing "enabled" to true. +2. Configure "refreshtimeoutinminutes" to be how long the lambda function will wait for trusted advisor to refresh the check data. +3. Add a check to the "checks" list by specifying Trusted Advisor named category and checkname. Important to specify the EXACT checkname and category as returned by the Trusted Advisor API. +4. Specify the account ID's you would like to enable this TA check on. +5. Specify any filters you'd like to set. A filter tells Trusted Advisor which results to return by looking at the metadata of the check response. For example, for Low Util EC2 you can turn on a filter that looks at the Estimated Monthly Savings attribute and will only return EC2 instances where the Estimated Monthly Savings is greater than $500. +* Attribute must be the name of the category you'd like to filter the check by. To see the attributes of a result, look at the Trusted Advisor API call, trusted-advisor-describe-checks. + +```$ aws support describe-trusted-advisor-checks --language en +``` +All possible checks are returned. The metadata section explains possible attributes for a specific check. + + +* Operator - "eq" is equal, "gt" is greater than. The operator compares the current value from the describe-checks-result api call to the value specified in the 'value' field. + +* Value - adjust depending on how you want to filter + +See an example of the **filter** section below: +``` +{ + "attribute": "Number of Days Low Utilization", + "operator": "eq", + "value": "14", + } +``` +If you do not want to filter the results, filters should be left as an empty list, like so: + +``` +filters: [], + +``` + +6. "ddb.table_name" is the name of the Dynamo Table that stores the issue details returned from a Trusted Advisor check. The table must exist before the check can be enabled. DDB tables are check specific. + +### 3.3. How to add your new check's DDB Table + +Add your all ddb information to deployment/cf-templates/ddb.json: +1. Update the reference name of the table +2. “Key Schema” + Must make the attributes account_id and issue_id +3. “BillingMode”: “PAY_PER_REQUEST” +4. “TableName”: + Specify your table name,which will follow the resources prefix. This is the name of the table that you will reference in the config file as the new check's ddb.table_name. + +### 3.4 Adding the checks to more Accounts +In config.json, go to the trusted_advisor_recommendations section, and add the account ID to each check object you want to enable on that account. Make sure that account has all updated permissions needed for Trusted Advisor checks. These permissions were updated in identificaiton-crossaccount-role.json. + +## 4. Understanding the initiate and describe functions +The initiate function gathers all specifications from the config file, makes them account specific, and fans out the information to the describe function, creating X number of describe instances for X number of accounts specified in config. + +account_and_checks: A dictionary to organize check information PER account. Each account object is taken in by a different describe instance. One account per describe instance. +``` + { + AccountID1: { + account_id: string, + checks_info: [{}], + client: account support client, + refresh_done: boolean + }, + AccountID2: { + account_id: string, + checks_info: [{}], + client: account support client, + refresh_done: boolean + } + } +``` +checks_info is a list of check objects: +``` + { + "checkname" : checkname, + "id" : check_id, + "name": name, + "category": category, + "metadata" : metadata, + "ddb.table_name" : ddb, + "filters": filters, + "refresh_done" : False + } +``` +In each describe instance, the function gathers the Trusted Advisor result for each specific check enabled for the Account. This is where the Trusted Advisor API call to **describe-trusted-advisor-check-result(checkID)** will be made to actually gather the results. Results may be filtered depending on the config settings. They are then sent to Dynamo DB for the specific check. + +## 5. Understanding Trusted Advisor API +Below will outline what the initiate and describe functions are doing with regard to making Trusted Advisor API calls. Running through this should give you a good idea of how Trsuted Advisor API works and how to get information you may need in enabling more checks with Trusted Advisor. + +### 5.1 Trusted Advisor API Calls: Simmulating Initiate and Describe + +The initiate function calls describe-trusted-advisor-checks, refresh-trusted-advisor-check, and describe-trusted-advisor-check-refresh-statuses. The describe function calls describe-trusted-advisor-check-result. + +Simmulate it in the CLI with the following commands: + +To gather all possible checks and their corresponding IDs: +``` +$ aws support describe-trusted-advisor-checks --language en +``` +To refresh a specific check's data: +``` +$aws support refresh-trusted-advisor-check --check-id Qch7DwouX1 +``` +To check the refresh status of the check: +``` +$ aws support describe-trusted-advisor-check-refresh-statuses --check-ids Qch7DwouX1 +``` +To get the official Trusted Advisor results of that check: +``` +$ aws support describe-trusted-advisor-check-result --check-id Qch7DwouX1 --language en +``` +## 6. Deployment in AWS Environment + +For your referance, Any {UPPERCASE LETTERS} in brackets is a variable. Fill appropriately. + +Make any applicable changes using above directions to deployment/configs/config.json + +``` +$ cd deployment +$ ./build_packages.sh configs/config.json +$ aws s3 sync packages/ {s3 bucket} +``` + +If you want to deploy through the cloudformation template, follow the below: + +### 6.1 Execution Role in master account +This command creates the master role in the master account. This is the role that attaches to the Lambdas and is used to assume the role in member accounts. + +``` +aws cloudformation deploy --template-file identification-role.json --stack-name {STACKNAME} --parameter-overrides IdentificationIAMRole={MASTER ROLE} IdentificationCrossAccountIAMRole={CROSS ACCOUNT ROLE} ResourcesPrefix={RESOURCES PREFIX} --capabilities CAPABILITY_NAMED_IAM --region us-east-1 +``` +### 6.2 Create StackSet for cross account role + +``` +aws cloudformation create-stack-set --stack-set-name {STACK SET NAME} --template-body file://identification-crossaccount-role.json --administration-role-arn {ADMIN ROLE ARN} --execution-role-name {EXECUTION ROLE} --parameters {ANY PARAMS} --capabilities CAPABILITY_NAMED_IAM --region {REGION} +``` +### 6.3 DynamoDB tables +This is a one-time command to initialize the ddb tables used in the master account. + +```aws cloudformation deploy --template-file ddb.json --stack-name {STACK NAME} --parameter-overrides ResourcesPrefix={PREFIX} --region {REGION} +``` +### 6.4 Identification Stack + +``` +aws cloudformation deploy --template-file identification.json --stack-name {STACKNAME} --parameter-overrides IdentificationIAMRole={MASTER ROLE} SourceS3Bucket={S3 BUCKET NAME} IdentificationCheckRateExpression="16 * * ? *" ResourcesPrefix={RESOURCES PREFIX} --s3-bucket {S3 BUCKET NAME} --s3-prefix {S3 PREFIX} --region {REGION} +``` + +### 6.5 Updating Lambda Code +To update lambda code, do all commands above to sync the s3 bucket and then run: + +``` +$ aws lambda update-function-code --function function-name --s3-bucket {s3 bucket} --s3-key scanzipfile +``` +An example of this, would be : +``` +$ aws lambda update-function-code --function dev-hammer-describe-ta-checks --{s3 bucket} --s3-key trusted-advisor-checks-identification.zip +``` + + + + diff --git a/docs/pages/playbook25_adding_a_rule.md b/docs/pages/playbook25_adding_a_rule.md new file mode 100644 index 00000000..ee35729a --- /dev/null +++ b/docs/pages/playbook25_adding_a_rule.md @@ -0,0 +1,97 @@ +--- +title: RDS unencrypted instances +keywords: playbook12 +sidebar: mydoc_sidebar +permalink: playbook12_rds_unencryption.html +--- + +# Playbook 25: Adding Your own Rule to Hammer + +## Introduction +The purpose of this document is to give an overview of the relevant files needed in the process of adding new checks to Hammer. The file changes outlined below are not necessarily in any specific order. The files may need to be revisited throughout the process as you add specifications. There may be more steps and file changes for your specific rule you are adding. Let this be a general guide to help get you started on integrating your new checks with Hammer! + +Hammer uses a nested stack model for deploying cloud formation templates. + + +## 1. Files +The following are files that will likely need to be updated for your specific rule. Read on to see what each means and how to update! + +deployment/build_packages.sh +deployment/cf-templates/ddb.json +deployment/cf-templates/identification-crossaccount-role.json +deployment/cf-templates/Identification-role.json +deployment/cf-templates/Identification.json +deployment/configs/config.json +hammer/identification/lambdas/ +hammer/identification/lambdas/metrics-publisher/metrics_publisher.py +hammer/library/aws/ +hammer/library/config.py +hammer/library/ddb_issues.py + + +## 2. Main Changes + + +1. Add DynamoDB information to ***deployment/cf-templates/ddb.json*** + +- Update the reference name of the table +- “Key Schema” +Must make the attributes account_id and issue_id +- “BillingMode”: “PAY_PER_REQUEST” +On demand payment makes more sense with how we are currently using the dynamo tables +- “TableName”: +Specify your table name,which will follow the resources prefix + +2. Update ***deployment/cf-templates/identification-crossaccount-role.json*** + +- This is the template for the CrossAcccount Role +- Holds the permissions and policies for HammerCrossAccountIdentifyRole +- CrossAccountRoles are assumed by the master account for permissions in slave accounts +- Change any policies/permissions as required +- Head to https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html, and find “JSON Policy Document Structure” to understand how to update your policies by adding a statement + + +3. Update ***deployment/cf-templates/Identification-role.json*** + +- This is the Master Role Cloud Formation template. +- Holds the permissions and policies for HammerMasterIdentifyRole +- This is the Master Role that all lambdas assume +- Update policies/permissions as needed + + +4. Update ***deployment/cf-templates/Identification.json*** + +- Identification stack for Hammer +- Nested Stack Template +- Add your rule’s Cloud Formation Stack following the conventions of previous stacks + +5. Add your rules configuration specifications to ***deployment/configs/config.json*** + +6. Update config parser ***hammer/library/config.py*** + +- utility functions to parse a rule’s specific config specifications +- Enable lambda to read config file you pass for your specific rule +- Can extend ModuleConfig or use ModuleConfig if your specific check config doesn’t have any extra properties to be accessed + +7. Add your lambda functions to ***hammer/identification/lambdas/*** + +- Create a folder for your rule and add all necessary lambda.py files : the initiate and describe functions. + +8. Add ddb name to ***hammer/identification/lambdas/metrics-publisher/metrics_publisher.py*** + +- Configure with ddb table name to publish metrics + +9. Add function methods to your specific check’s util file (create one if applicable) in ***hammer/library/aws/*** + +- Parent folder that holds all rule’s utility functions. +- Add logic to the Checker class, which checks the status of resources in AWS environ +- EXAMPLE) hammer/library/aws/ta.py holds the trusted advisor helper functions including Checker class + +10. Update ***hammer/library/ddb_issues.py*** +- Add your specific rule Issue Class +- May need to provide it specific functionality depending on your rule + +11. Add lambda folder name to ***deployment/build_packages.sh*** +- Under LAMBDAS variable, add the name of the folder for your new rule. This is the folder that will be zipped and deployed. + + diff --git a/hammer/identification/lambdas/trusted-advisor-checks-identification/README.md b/hammer/identification/lambdas/trusted-advisor-checks-identification/README.md new file mode 100644 index 00000000..b16478c5 --- /dev/null +++ b/hammer/identification/lambdas/trusted-advisor-checks-identification/README.md @@ -0,0 +1,193 @@ +# Trusted Advisor Checks + +Enabling Trusted Advisor checks with Hammer allows you to add all rules and checks from AWS’s Trusted Advisor with Dow Jones’ Hammer Continuous Monitoring Service. + +File Structure: + +``` + +├── trusted-advisor-checks-identification <-- Parent folder for Trusted Advisor Checks +│ ├── README.md <-- This Instructions file +│   ├── initiate-ta-checks.py <-- Initiate Lambda function +│   ├── describe-ta-checks.py <-- Describe Lambda function + +``` + +Configuration Specification: + +``` +{ + "trusted_advisor_recommendations": { + "enabled": true, + "refreshtimeoutinminutes": 8, + "checks": [ + { + "category": "cost_optimizing", + "checkname": "Low Utilization Amazon EC2 Instances", + "name": "trusted_advisor_recommendations_low_ec2_utilization", + "accounts": [ + "Account ID 1", + "Account ID 2" + ], + "filters": [ + { + "attribute": "Estimated Monthly Savings", + "operator": "gt", + "value": "500" + }, + { + "attribute": "Number of Days Low Utilization", + "operator": "eq", + "value": "14", + } + ], + "ddb.table_name": "hammer-trusted-advisor-cost-optimizing-low-utilization-ec2-instances" + }, + { + "category": "fault_tolerance", + "checkname": "Amazon EBS Snapshots", + "accounts": [ + "Account 1", + "Account 2" + ], + "filters": [ + { + "attribute": "status", + "operator": "eq", + "value": "critical" + } + ], + "ddb.table_name": "hammer-trusted-advisor-fault-tolerance-ebs-snapshots" + } + ] + } +} +``` + +#How to configure Trusted Advisor checks: +1. Turn on Trusted Advisor checks by changing "enabled" to true. +2. "refreshtimeoutinminutes" is how long the initiate lambda function will wait for trusted advisor to refresh the check data. +3. Add a check to the "checks" list by specifying Trusted Advisor named category and checkname. Important to specify the EXACT checkname and category as returned by the Trusted Advisor API. +4. Specify the account ID's you would like to enable this TA check on. +5. Specify any filters you'd like to set. A filter tells Trusted Advisor which results to return by looking at the metadata of the check response. For example, for Low Util EC2 you can turn on a filter that looks at the Estimated Monthly Savings attribute and will only return EC2 instances where the Estimated Monthly Savings is greater than $500. + +* Attribute must be the name of the category you'd like to filter the check by. To see the attributes of a result, look at the Trusted Advisor API call, trusted-advisor-describe-checks. All possible checks are returned. The metadata section explains possible attributes for a specific check. + +```$ aws support describe-trusted-advisor-checks --language en +``` + +* Operator - "eq" is equal, "gt" is greater than. The operator compares the current value from the describe-checks-result api call to the value specified in the 'value' field. + +* Value - adjust depending on how you want to filter +``` +{ + "attribute": "Number of Days Low Utilization", + "operator": "eq", + "value": "14", + } +``` +If you do not want to filter the results, filters should be left as an empty list, like so: + +``` +filters: [], + +``` + +6. "ddb.table_name" is the name of the Dynamo Table that stores the issue details returned from a Trusted Advisor checks. The table must exist before the check can be enabled. DDB tables are check specific. + +#How to add your new check's DDB Table: +Add your all ddb information to deployment/cf-templates/ddb.json: +1. Update the reference name of the table +2. “Key Schema” + Must make the attributes account_id and issue_id +3. “BillingMode”: “PAY_PER_REQUEST” +4. “TableName”: + Specify your table name,which will follow the resources prefix. This is the name of the table that you will reference in the config file as the new check's ddb.table_name. + + +#Understanding initiate-ta-checks.py: +The initiate function gathers all specifications from the config file, makes them account specific, and fans out the information to the describe function. The describe function evaluates the checks for ONE account. In other words, one account per describe instance. + +account_and_checks: A dictionary to organize check information PER account. +``` + { + AccountID1: { + account_id: string + checks_info: [{}], + session: account session variable + }, + AccountID2: { + account_id: string + checks_info: [{}], + session: account session variable, + refresh_done: boolean + } + } +``` +checks_info is a list of check objects: +``` + { + "checkname" : checkname, + "id" : check_id, + "name": name, + "category": category, + "metadata" : metadata, + "ddb.table_name" : ddb, + "filters": filters, + "refresh_done" : False + } +``` +#Understanding describe-ta-checks.py: + +Gathers and filters the Trusted Advisor result for each specific check enabled for the Account. + +#Understanding Trusted Advisor API +Trusted Advisor API Calls +The initiate function calls describe-trusted-advisor-checks, refresh-trusted-advisor-check, and describe-trusted-advisor-check-refresh-statuses. The describe function calls describe-trusted-advisor-check-result. + +To simmulate it with the CLI follow below commands: + +To gather all possible checks and their corresponding IDs: +``` +$ aws support describe-trusted-advisor-checks --language en +``` +To refresh a specific check's data: +``` +$aws support refresh-trusted-advisor-check --check-id Qch7DwouX1 +``` +To check the refresh status of the check: +``` +$ aws support describe-trusted-advisor-check-refresh-statuses --check-ids Qch7DwouX1 +``` +To get the official Trusted Advisor results of that check: +``` +$ aws support describe-trusted-advisor-check-result --check-id Qch7DwouX1 --language en +``` +#How to Configure More Slave Accounts for TA Checks +In config.json, go to the trsuted_advisor_recommendations section, and add the account ID for each check you want to enable on that account. Make sure that account has all updated permissions needed for Trusted Advisor checks. These permissions were updated in identificaiton-crossaccount-role.json + +#To deploy in AWS Environment through Cloud Formation Templates + +Make any applicable changes using above directions to deployment/configs/config-tadev.json + +``` +$ cd deployment +$ ./build_packages.sh configs/config.json +$ aws s3 sync packages/ s3://{s3 bucket name} + +$ cd cf-templates +$ aws cloudformation deploy --template-file identification.json --stack-name New-Test-CloudSecurity-Hammer-Identification-Dev-Stack --parameter-overrides IdentificationIAMRole=new-test-cloudsecurity-master-role NestedStackTemplate=https://ta-checks-test.s3-us-west-2.amazonaws.com/cf-templates/identification-nested.json SourceS3Bucket=ta-checks-test IdentificationCheckRateExpression="16 * * ? *" ResourcesPrefix="dev-hammer-test" --s3-bucket ta-checks-test --s3-prefix cf-templates --region us-west-2 + +``` +To update lambda code, do the first 3 commands above to sync the s3 bucket and then run: + +``` +$ aws lambda update-function-code --function function-name --s3-bucket {s3 bucket} --s3-key scanzipfile +``` +An example of this, would be : +``` +$ aws lambda update-function-code --function dev-hammer-describe-ta-checks --{s3 bucket} --s3-key trusted-advisor-checks-identification.zip +``` + + + diff --git a/hammer/identification/lambdas/trusted-advisor-checks-identification/describe_ta_checks.py b/hammer/identification/lambdas/trusted-advisor-checks-identification/describe_ta_checks.py new file mode 100755 index 00000000..20e73a9b --- /dev/null +++ b/hammer/identification/lambdas/trusted-advisor-checks-identification/describe_ta_checks.py @@ -0,0 +1,75 @@ +import json +import logging +import os + +from library.logger import set_logging +from library.config import Config +from library.aws.ta import TrustedAdvisorChecker +from library.aws.utility import Account +from library.ddb_issues import IssueStatus +from library.ddb_issues import TACheckIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import Sns + +def lambda_handler(event, context): + """ Lambda handler to evaluate trusted advisor checks """ + set_logging(level=logging.DEBUG) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + checks = payload['checks'] + + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + main_account = Account(region=config.aws.region) + account = Account(id=account_id, role_name=config.aws.role_name_identification, region="us-east-1") + + for check in checks: + check_id = check["id"] + + ddb_table_name = check["ddb.table_name"] + filters = check["filters"] + metadata = check["metadata"] + ddb_table = main_account.resource("dynamodb").Table(ddb_table_name) + category = check["category"] + checkname = check["checkname"] + + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, TACheckIssue) + open_issues = {issue.issue_id: issue for issue in open_issues} + logging.debug(f"TA in DDB:\n{open_issues.keys()}") + + checker = TrustedAdvisorChecker(account=account, check_id=check_id) + + #grab the trusted advisor response unfiltered + result = checker.get_ta_result() + filtered = False + if filters: + filtered = True + result = checker.filter(filters, metadata, result) + for vuln_resource in result["result"]["flaggedResources"]: + resource_id = vuln_resource["resourceId"] + issue = TACheckIssue(account_id,resource_id) + issue.issue_details.status = vuln_resource["status"] + issue.issue_details.filtered = filtered + issue.issue_details.metadata = vuln_resource["metadata"] + issue.issue_details.region = vuln_resource["region"] + issue.issue_details.checkname = checkname + issue.issue_details.category = category + issue.status = IssueStatus.Open + logging.debug("Setting status to Open") + IssueOperations.update(ddb_table, issue) + + open_issues.pop(resource_id, None) + + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + + + except Exception: + logging.exception(f"Failed to check Trusted Advisor findings for '{account_id}'") + return diff --git a/hammer/identification/lambdas/trusted-advisor-checks-identification/initiate_to_desc_ta_checks.py b/hammer/identification/lambdas/trusted-advisor-checks-identification/initiate_to_desc_ta_checks.py new file mode 100755 index 00000000..153ec863 --- /dev/null +++ b/hammer/identification/lambdas/trusted-advisor-checks-identification/initiate_to_desc_ta_checks.py @@ -0,0 +1,97 @@ +import os +import logging +import time + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns +from library.aws.utility import Account + +def lambda_handler(event, context): + """ Lambda handler to initiate to find SQS public access in policy """ + set_logging(level=logging.INFO) + logging.debug("Initiating SQS policies checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.trustedAdvisor.enabled: + logging.debug("TA policies checking disabled") + return + + except Exception: + logging.exception("Failed to parse config") + return + + logging.debug("Iterating over each check to grab specified accounts") + + account_and_checks = {} + + for config_check in config.trustedAdvisor.checks: + list_of_accounts_per_check = config_check["accounts"] + checkname = config_check["checkname"] + ddb = config_check["ddb.table_name"] + filters = config_check["filters"] + name = config_check["name"] + category = config_check["category"] + + for account_id in list_of_accounts_per_check: + + if (account_and_checks.get(account_id) is None): + + client = Account(id=account_id, region="us-east-1").client("support") + account_and_checks.update({account_id: {"account_id" : account_id, "client": client, "checks_info": []} + }) + + check_response = account_and_checks.get(account_id).get("client").describe_trusted_advisor_checks(language='en') + + for ck in check_response["checks"]: + if ck["name"] == checkname: + check_info = ck + try: + check_id = check_info['id'] + metadata = check_info['metadata'] + check_id_obj = {"checkname" : checkname, "id" : check_id, "name": name, "category": category, "metadata" : metadata, "ddb.table_name" : ddb, "filters": filters, "refresh_done" : False} + account_and_checks.get(account_id)["checks_info"].append(check_id_obj) + + except: + logging.exception("No check to match checkname given") + return + account_and_checks.get(account_id).get("client").refresh_trusted_advisor_check(checkId=check_id) + + timeout = config.trustedAdvisor.refreshtimeout*60 + start = time.time() + + while True: + all_refreshes_done = True + for acct in account_and_checks: + current_account_obj = account_and_checks[acct] + for chk in current_account_obj["checks_info"]: + if chk["refresh_done"] == False: + status = current_account_obj["client"].describe_trusted_advisor_check_refresh_statuses(checkIds=[chk["id"]]) + if not status["statuses"][0]["status"] == "success": + all_refreshes_done = False + else: + chk["refresh_done"] = True + else: + break + end = time.time() + if all_refreshes_done == True or end - start >= timeout: + break + + for account in account_and_checks: + try: + acc = account_and_checks[account] + payload = { "account_id": acc["account_id"], + "checks": acc["checks_info"] + } + logging.debug("Initiating TA describe") + Sns.publish(sns_arn, payload) + + except Exception: + logging.exception("Error occurred while initiation of SNS") + return + + logging.debug("TA initiation done") + diff --git a/hammer/library/aws/ta.py b/hammer/library/aws/ta.py new file mode 100755 index 00000000..3ad4ac71 --- /dev/null +++ b/hammer/library/aws/ta.py @@ -0,0 +1,105 @@ +import json +import logging +import mimetypes +import pathlib +from library.config import Config + + +from datetime import datetime, timezone +from io import BytesIO +from copy import deepcopy +from botocore.exceptions import ClientError +from library.utility import jsonDumps +from library.aws.utility import convert_tags + + +class TrustedAdvisorChecker(object): + """ + Basic class for gathering trusted advisor checks in account. + """ + def __init__(self, account, check_id): + """ + :param account: `Account` instance to grab trusted advisor findings for. + :param checkID: unique check identification used to grab corresponding results. + """ + self.account = account + self.client = account.client("support") + self.check_id = check_id + + def filter(self, filters, name_metadata, result): + """ + Apply filters for a specific check. + """ + + resources = result["result"]["flaggedResources"] + + int = 0 + while int < len(resources): + organized_metadata = {} + curr = 0 + for val in name_metadata: + organized_metadata[val] = resources[int]["metadata"][curr] + curr += 1 + resources[int]["metadata"] = organized_metadata + for filt in filters: + #apply all the filters to this resource + attribute = filt["attribute"] + standard_value = filt["value"] + op = filt["operator"] + + current_resource_value = organized_metadata[attribute] + new_val = self.parse_value(current_resource_value, attribute) + keep = self.compare(new_val, standard_value, op) + if not keep: + resources.pop(int) + result["result"]["resourcesSummary"]["resourcesIgnored"] += 1 + #stop checking + break + if keep: + int += 1 + return result + + def parse_value(self, resource_val, filter_type): + """ + Remove unnecessary values from string to enable comparison to config filters. + + :param resource_val: the string to parse, returned from trusted advisor findings. + :param filter_type: The data to filter by, specified in the config file as the attribute. + + """ + parsed = resource_val + if filter_type == "Estimated Monthly Savings": + parsed = resource_val.replace("$", '') + if filter_type == "Number of Days Low Utilization": + parsed = resource_val.replace(" days", '') + return parsed + + def compare(self, new_value, standard, operator): + """ + Returns true if the new_value passes the constraints. By default return True. + :param new_value: the parsed value returned from Trusted Advisor findings of a specific attribute. + :param standarad: the filter condition specified in config's filter option. + :param operator: specifies how the new value should be compared to the standard. + + OPTIONS:"gt": greater than + "eq": equal + """ + if operator == "gt": + return float(new_value) > float(standard) + if operator == "eq": + return new_value == standard + return True + + def get_ta_result(self): + """ + Call the Trusted Advisor API to describe check results for specified check. Filter and enrich if necessary. + + :param checkId: the ID of the Trusted Advisor check, needed to gather all Trusted Advisor findings. + + :return: a dictionary containing the Trusted Advisor api response + """ + try: + response = self.client.describe_trusted_advisor_check_result(checkId=self.check_id, language="en") + return response + except: + logging.exception(f"Failed to call Trusted Advisor initial findings.") diff --git a/hammer/library/config.py b/hammer/library/config.py index 11ad9eee..e745e418 100755 --- a/hammer/library/config.py +++ b/hammer/library/config.py @@ -105,6 +105,8 @@ def __init__(self, self.slack = SlackConfig(slack_config) # CSV configuration self.csv = CSVConfig(self._config, self.slack) + #Trusted Advisor Config + self.trustedAdvisor = TrustedAdvisorConfig(self._config, "trusted_advisor_recommendations") # API configuration self.api = ApiConfig({ @@ -587,3 +589,15 @@ class IAMUserKeysRotationConfig(ModuleConfig): def rotation_criteria_days(self): """ :return: `timedelta` object to compare and mark access keys as stale (created long time ago) """ return timedelta(days=int(self._config["rotation_criteria_days"])) + +class TrustedAdvisorConfig(ModuleConfig): + """Extend ModuleConfig with TrustedAdvisor specific properties """ + @property + def refreshtimeout(self): + """ return int of the max time to run function, default is 10 minutes""" + return int(self._config.get("refreshtimeoutinminutes", 10)) + @property + def checks(self): + """ Return a list of check objects. + """ + return self._config.get("checks", []) diff --git a/hammer/library/ddb_issues.py b/hammer/library/ddb_issues.py index 06cf1c4b..590dd079 100755 --- a/hammer/library/ddb_issues.py +++ b/hammer/library/ddb_issues.py @@ -252,7 +252,7 @@ class RedshiftPublicAccessIssue(Issue): def __init__(self, *args): super().__init__(*args) - + class ECSLoggingIssue(Issue): def __init__(self, *args): super().__init__(*args) @@ -272,17 +272,21 @@ class ESEncryptionIssue(Issue): def __init__(self, *args): super().__init__(*args) - + class ESLoggingIssue(Issue): def __init__(self, *args): super().__init__(*args) - + class ESPublicAccessIssue(Issue): def __init__(self, *args): - super().__init__(*args) + super().__init__(*args) + +class TACheckIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + - class Operations(object): @staticmethod def find(ddb_table, issue):