From 227a801f08085644e469c8464234d866f0babf20 Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Mon, 8 Jul 2024 15:29:57 +0200 Subject: [PATCH 1/5] feat: Update AWS Lambda functions runtime to Python 3.12, set their AWS CloudWatch Log Groups retention period to 14 days, make IAM policy actions for AWS CloudWatch more restrictive and refactor the code. --- backup_function.tf | 75 ++++++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/backup_function.tf b/backup_function.tf index 4a6fb48..3150ca2 100644 --- a/backup_function.tf +++ b/backup_function.tf @@ -20,7 +20,7 @@ EOF resource "aws_iam_role_policy" "ebs_backup_policy" { name = "ebs_backup_policy" - role = "${aws_iam_role.ebs_backup_role.id}" + role = aws_iam_role.ebs_backup_role.id policy = < Date: Mon, 8 Jul 2024 15:32:59 +0200 Subject: [PATCH 2/5] feat: Update Python scripts to make them compatible with Python 3.12, refactor them and fix linting errors in them. --- ...shot-janitor.py => ebs_snapshot_janitor.py | 45 ++++++++----------- ...ups.py => schedule_ebs_snapshot_backups.py | 43 +++++++++--------- 2 files changed, 41 insertions(+), 47 deletions(-) rename ebs-snapshot-janitor.py => ebs_snapshot_janitor.py (52%) rename schedule-ebs-snapshot-backups.py => schedule_ebs_snapshot_backups.py (59%) diff --git a/ebs-snapshot-janitor.py b/ebs_snapshot_janitor.py similarity index 52% rename from ebs-snapshot-janitor.py rename to ebs_snapshot_janitor.py index b2159ef..2f30e8d 100644 --- a/ebs-snapshot-janitor.py +++ b/ebs_snapshot_janitor.py @@ -11,43 +11,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import boto3 +# pylint: disable=missing-module-docstring import re import datetime +import boto3 -ec = boto3.client('ec2') -iam = boto3.client('iam') +def lambda_handler(event, context): # pylint: disable=unused-argument + """ + This function looks at all the snapshots of AWS EBS volumes that have a + "DeleteOn" tag containing the current day formatted as YYYY-MM-DD. + This function should be run at least daily. + """ + account_ids = [] -""" -This function looks at *all* snapshots that have a "DeleteOn" tag containing -the current day formatted as YYYY-MM-DD. This function should be run at least -daily. -""" + ec2_client = boto3.client('ec2') + iam_client = boto3.client('iam') -def lambda_handler(event, context): - account_ids = list() try: - """ - You can replace this try/except by filling in `account_ids` yourself. - Get your account ID with: - > import boto3 - > iam = boto3.client('iam') - > print iam.get_user()['User']['Arn'].split(':')[4] - """ - iam.get_user() - except Exception as e: + iam_client.get_user() + except Exception as exception: # pylint: disable=broad-except # use the exception message to get the account ID the function executes under - account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(e)).groups()[1]) - + account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(exception)).groups()[1]) delete_on = datetime.date.today().strftime('%Y-%m-%d') filters = [ {'Name': 'tag-key', 'Values': ['DeleteOn']}, {'Name': 'tag-value', 'Values': [delete_on]}, ] - snapshot_response = ec.describe_snapshots(OwnerIds=account_ids, Filters=filters) - + snapshot_response = ec2_client.describe_snapshots(OwnerIds=account_ids, Filters=filters) - for snap in snapshot_response['Snapshots']: - print "Deleting snapshot %s" % snap['SnapshotId'] - ec.delete_snapshot(SnapshotId=snap['SnapshotId']) + for snapshot in snapshot_response['Snapshots']: + snapshot_id = snapshot['SnapshotId'] + print(f"Deleting snapshot {snapshot_id}") + ec2_client.delete_snapshot(SnapshotId=snapshot_id) diff --git a/schedule-ebs-snapshot-backups.py b/schedule_ebs_snapshot_backups.py similarity index 59% rename from schedule-ebs-snapshot-backups.py rename to schedule_ebs_snapshot_backups.py index ab201f0..1b4d8c8 100644 --- a/schedule-ebs-snapshot-backups.py +++ b/schedule_ebs_snapshot_backups.py @@ -11,14 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import boto3 +# pylint: disable=missing-module-docstring import collections import datetime +import boto3 -ec = boto3.client('ec2') +def lambda_handler(event, context): # pylint: disable=unused-argument + """ + This function creates snapshots of AWS EBS volumes attached to AWS EC2 instances + that have a "Backup" tag containing either 'true', 'yes', or '1'. + This function should be run at least daily. + """ + ec2_client = boto3.client('ec2') -def lambda_handler(event, context): - reservations = ec.describe_instances( + reservations = ec2_client.describe_instances( Filters=[ {'Name': 'tag:Backup', 'Values': ['true', 'yes', '1']}, ] @@ -26,13 +32,11 @@ def lambda_handler(event, context): 'Reservations', [] ) - instances = sum( - [ - [i for i in r['Instances']] - for r in reservations - ], []) + instances = [ + instance for reservation in reservations for instance in reservation['Instances'] + ] - print "Found %d instances that need backing up" % len(instances) + print(f"Found {str(len(instances))} instances that need backing up") to_tag = collections.defaultdict(list) @@ -48,28 +52,25 @@ def lambda_handler(event, context): if dev.get('Ebs', None) is None: continue vol_id = dev['Ebs']['VolumeId'] - print "Found EBS volume %s on instance %s" % ( - vol_id, instance['InstanceId']) + print(f"Found EBS volume {vol_id} on instance {instance['InstanceId']}") - snap = ec.create_snapshot( + snap = ec2_client.create_snapshot( VolumeId=vol_id, ) to_tag[retention_days].append(snap['SnapshotId']) - print "Retaining snapshot %s of volume %s from instance %s for %d days" % ( - snap['SnapshotId'], - vol_id, - instance['InstanceId'], - retention_days, + print( + f"Retaining snapshot {snap['SnapshotId']} of volume {vol_id} " + f"from instance {instance['InstanceId']} for {str(retention_days)} days" ) - for retention_days in to_tag.keys(): + for retention_days in to_tag.keys(): # pylint: disable=consider-using-dict-items delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) delete_fmt = delete_date.strftime('%Y-%m-%d') - print "Will delete %d snapshots on %s" % (len(to_tag[retention_days]), delete_fmt) - ec.create_tags( + print(f"Will delete {str(len(to_tag[retention_days]))} snapshots on {delete_fmt}") + ec2_client.create_tags( Resources=to_tag[retention_days], Tags=[ {'Key': 'DeleteOn', 'Value': delete_fmt}, From 0e79bd6385555467490d16463998efbfdd411dc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abdullah=20Khawer=20=F0=9F=87=B5=F0=9F=87=B0=20?= =?UTF-8?q?=F0=9F=87=B3=F0=9F=87=B1=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Mon, 8 Jul 2024 15:47:29 +0200 Subject: [PATCH 3/5] Revert "feat: Upgrade Python code to Python 3.12 and refactor the whole code and make improvements" --- backup_function.tf | 75 +++++++------------ ...shot_janitor.py => ebs-snapshot-janitor.py | 45 ++++++----- ...ups.py => schedule-ebs-snapshot-backups.py | 43 ++++++----- 3 files changed, 73 insertions(+), 90 deletions(-) rename ebs_snapshot_janitor.py => ebs-snapshot-janitor.py (52%) rename schedule_ebs_snapshot_backups.py => schedule-ebs-snapshot-backups.py (59%) diff --git a/backup_function.tf b/backup_function.tf index 3150ca2..4a6fb48 100644 --- a/backup_function.tf +++ b/backup_function.tf @@ -20,7 +20,7 @@ EOF resource "aws_iam_role_policy" "ebs_backup_policy" { name = "ebs_backup_policy" - role = aws_iam_role.ebs_backup_role.id + role = "${aws_iam_role.ebs_backup_role.id}" policy = < import boto3 + > iam = boto3.client('iam') + > print iam.get_user()['User']['Arn'].split(':')[4] + """ + iam.get_user() + except Exception as e: # use the exception message to get the account ID the function executes under - account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(exception)).groups()[1]) + account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(e)).groups()[1]) + delete_on = datetime.date.today().strftime('%Y-%m-%d') filters = [ {'Name': 'tag-key', 'Values': ['DeleteOn']}, {'Name': 'tag-value', 'Values': [delete_on]}, ] - snapshot_response = ec2_client.describe_snapshots(OwnerIds=account_ids, Filters=filters) + snapshot_response = ec.describe_snapshots(OwnerIds=account_ids, Filters=filters) + - for snapshot in snapshot_response['Snapshots']: - snapshot_id = snapshot['SnapshotId'] - print(f"Deleting snapshot {snapshot_id}") - ec2_client.delete_snapshot(SnapshotId=snapshot_id) + for snap in snapshot_response['Snapshots']: + print "Deleting snapshot %s" % snap['SnapshotId'] + ec.delete_snapshot(SnapshotId=snap['SnapshotId']) diff --git a/schedule_ebs_snapshot_backups.py b/schedule-ebs-snapshot-backups.py similarity index 59% rename from schedule_ebs_snapshot_backups.py rename to schedule-ebs-snapshot-backups.py index 1b4d8c8..ab201f0 100644 --- a/schedule_ebs_snapshot_backups.py +++ b/schedule-ebs-snapshot-backups.py @@ -11,20 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=missing-module-docstring +import boto3 import collections import datetime -import boto3 -def lambda_handler(event, context): # pylint: disable=unused-argument - """ - This function creates snapshots of AWS EBS volumes attached to AWS EC2 instances - that have a "Backup" tag containing either 'true', 'yes', or '1'. - This function should be run at least daily. - """ - ec2_client = boto3.client('ec2') +ec = boto3.client('ec2') - reservations = ec2_client.describe_instances( +def lambda_handler(event, context): + reservations = ec.describe_instances( Filters=[ {'Name': 'tag:Backup', 'Values': ['true', 'yes', '1']}, ] @@ -32,11 +26,13 @@ def lambda_handler(event, context): # pylint: disable=unused-argument 'Reservations', [] ) - instances = [ - instance for reservation in reservations for instance in reservation['Instances'] - ] + instances = sum( + [ + [i for i in r['Instances']] + for r in reservations + ], []) - print(f"Found {str(len(instances))} instances that need backing up") + print "Found %d instances that need backing up" % len(instances) to_tag = collections.defaultdict(list) @@ -52,25 +48,28 @@ def lambda_handler(event, context): # pylint: disable=unused-argument if dev.get('Ebs', None) is None: continue vol_id = dev['Ebs']['VolumeId'] - print(f"Found EBS volume {vol_id} on instance {instance['InstanceId']}") + print "Found EBS volume %s on instance %s" % ( + vol_id, instance['InstanceId']) - snap = ec2_client.create_snapshot( + snap = ec.create_snapshot( VolumeId=vol_id, ) to_tag[retention_days].append(snap['SnapshotId']) - print( - f"Retaining snapshot {snap['SnapshotId']} of volume {vol_id} " - f"from instance {instance['InstanceId']} for {str(retention_days)} days" + print "Retaining snapshot %s of volume %s from instance %s for %d days" % ( + snap['SnapshotId'], + vol_id, + instance['InstanceId'], + retention_days, ) - for retention_days in to_tag.keys(): # pylint: disable=consider-using-dict-items + for retention_days in to_tag.keys(): delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) delete_fmt = delete_date.strftime('%Y-%m-%d') - print(f"Will delete {str(len(to_tag[retention_days]))} snapshots on {delete_fmt}") - ec2_client.create_tags( + print "Will delete %d snapshots on %s" % (len(to_tag[retention_days]), delete_fmt) + ec.create_tags( Resources=to_tag[retention_days], Tags=[ {'Key': 'DeleteOn', 'Value': delete_fmt}, From 9cb5ad249f921d435a3b3a98dbe4e313fc0e37c6 Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Mon, 8 Jul 2024 15:48:55 +0200 Subject: [PATCH 4/5] feat: Update AWS Lambda functions runtime to Python 3.12, set their AWS CloudWatch Log Groups retention period to 14 days, make IAM policy actions for AWS CloudWatch more restrictive and refactor the code. --- backup_function.tf | 75 ++++++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/backup_function.tf b/backup_function.tf index 4a6fb48..3150ca2 100644 --- a/backup_function.tf +++ b/backup_function.tf @@ -20,7 +20,7 @@ EOF resource "aws_iam_role_policy" "ebs_backup_policy" { name = "ebs_backup_policy" - role = "${aws_iam_role.ebs_backup_role.id}" + role = aws_iam_role.ebs_backup_role.id policy = < Date: Mon, 8 Jul 2024 15:49:09 +0200 Subject: [PATCH 5/5] feat: Update Python scripts to make them compatible with Python 3.12, refactor them and fix linting errors in them. --- ...shot-janitor.py => ebs_snapshot_janitor.py | 38 ++++++++-------- ...ups.py => schedule_ebs_snapshot_backups.py | 44 +++++++++---------- 2 files changed, 41 insertions(+), 41 deletions(-) rename ebs-snapshot-janitor.py => ebs_snapshot_janitor.py (59%) rename schedule-ebs-snapshot-backups.py => schedule_ebs_snapshot_backups.py (59%) diff --git a/ebs-snapshot-janitor.py b/ebs_snapshot_janitor.py similarity index 59% rename from ebs-snapshot-janitor.py rename to ebs_snapshot_janitor.py index b2159ef..8c093c4 100644 --- a/ebs-snapshot-janitor.py +++ b/ebs_snapshot_janitor.py @@ -11,21 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import boto3 +# pylint: disable=missing-module-docstring import re import datetime +import boto3 -ec = boto3.client('ec2') -iam = boto3.client('iam') +def lambda_handler(event, context): # pylint: disable=unused-argument + """ + This function looks at all the snapshots of AWS EBS volumes that have a + "DeleteOn" tag containing the current day formatted as YYYY-MM-DD. + This function should be run at least daily. + """ + account_ids = [] -""" -This function looks at *all* snapshots that have a "DeleteOn" tag containing -the current day formatted as YYYY-MM-DD. This function should be run at least -daily. -""" + ec2_client = boto3.client('ec2') + iam_client = boto3.client('iam') -def lambda_handler(event, context): - account_ids = list() try: """ You can replace this try/except by filling in `account_ids` yourself. @@ -34,20 +35,19 @@ def lambda_handler(event, context): > iam = boto3.client('iam') > print iam.get_user()['User']['Arn'].split(':')[4] """ - iam.get_user() - except Exception as e: + iam_client.get_user() + except Exception as exception: # pylint: disable=broad-except # use the exception message to get the account ID the function executes under - account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(e)).groups()[1]) - + account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(exception)).groups()[1]) delete_on = datetime.date.today().strftime('%Y-%m-%d') filters = [ {'Name': 'tag-key', 'Values': ['DeleteOn']}, {'Name': 'tag-value', 'Values': [delete_on]}, ] - snapshot_response = ec.describe_snapshots(OwnerIds=account_ids, Filters=filters) - + snapshot_response = ec2_client.describe_snapshots(OwnerIds=account_ids, Filters=filters) - for snap in snapshot_response['Snapshots']: - print "Deleting snapshot %s" % snap['SnapshotId'] - ec.delete_snapshot(SnapshotId=snap['SnapshotId']) + for snapshot in snapshot_response['Snapshots']: + snapshot_id = snapshot['SnapshotId'] + print(f"Deleting snapshot {snapshot_id}") + ec2_client.delete_snapshot(SnapshotId=snapshot_id) diff --git a/schedule-ebs-snapshot-backups.py b/schedule_ebs_snapshot_backups.py similarity index 59% rename from schedule-ebs-snapshot-backups.py rename to schedule_ebs_snapshot_backups.py index ab201f0..971cfd1 100644 --- a/schedule-ebs-snapshot-backups.py +++ b/schedule_ebs_snapshot_backups.py @@ -11,14 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import boto3 +# pylint: disable=missing-module-docstring import collections import datetime +import boto3 -ec = boto3.client('ec2') +def lambda_handler(event, context): # pylint: disable=unused-argument + """ + This function creates snapshots of AWS EBS volumes attached to AWS EC2 instances + that have a "Backup" tag containing either 'true', 'yes', or '1'. + This function should be run at least daily. + """ + ec2_client = boto3.client('ec2') -def lambda_handler(event, context): - reservations = ec.describe_instances( + reservations = ec2_client.describe_instances( Filters=[ {'Name': 'tag:Backup', 'Values': ['true', 'yes', '1']}, ] @@ -26,13 +32,11 @@ def lambda_handler(event, context): 'Reservations', [] ) - instances = sum( - [ - [i for i in r['Instances']] - for r in reservations - ], []) + instances = [ + instance for reservation in reservations for instance in reservation['Instances'] + ] - print "Found %d instances that need backing up" % len(instances) + print(f"Found {str(len(instances))} instances that need backing up") to_tag = collections.defaultdict(list) @@ -48,28 +52,24 @@ def lambda_handler(event, context): if dev.get('Ebs', None) is None: continue vol_id = dev['Ebs']['VolumeId'] - print "Found EBS volume %s on instance %s" % ( - vol_id, instance['InstanceId']) + print(f"Found EBS volume {vol_id} on instance {instance['InstanceId']}") - snap = ec.create_snapshot( + snap = ec2_client.create_snapshot( VolumeId=vol_id, ) to_tag[retention_days].append(snap['SnapshotId']) - print "Retaining snapshot %s of volume %s from instance %s for %d days" % ( - snap['SnapshotId'], - vol_id, - instance['InstanceId'], - retention_days, + print( + f"Retaining snapshot {snap['SnapshotId']} of volume {vol_id} " + f"from instance {instance['InstanceId']} for {str(retention_days)} days" ) - - for retention_days in to_tag.keys(): + for retention_days in to_tag.keys(): # pylint: disable=consider-using-dict-items delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) delete_fmt = delete_date.strftime('%Y-%m-%d') - print "Will delete %d snapshots on %s" % (len(to_tag[retention_days]), delete_fmt) - ec.create_tags( + print(f"Will delete {str(len(to_tag[retention_days]))} snapshots on {delete_fmt}") + ec2_client.create_tags( Resources=to_tag[retention_days], Tags=[ {'Key': 'DeleteOn', 'Value': delete_fmt},