AWSTemplateFormatVersion: '2010-09-09'
Description: >
  ElastiCache Snapshot Review Remediator.
  Purpose-specific, opt-in remediation module for reviewing ElastiCache clusters,
  replication groups, and manual snapshots. Optional remediation deletes explicitly
  approved old manual snapshots only. It does not delete clusters or replication groups.

Parameters:
  ProjectName:
    Type: String
    Default: elasticache-snapshot-review
    Description: Short base name used for created resources.

  TargetRegionsCsv:
    Type: String
    Default: all-enabled
    Description: Comma-separated AWS Regions to scan, or all-enabled to scan all enabled Regions.

  ExcludedRegionsCsv:
    Type: String
    Default: ap-northeast-3
    Description: Comma-separated AWS Regions to exclude from scanning/remediation.

  DryRun:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: When true, only report what would happen. When false, perform eligible approved snapshot deletion.

  ActionMode:
    Type: String
    Default: review-only
    AllowedValues:
      - review-only
      - delete-approved-manual-snapshots
    Description: review-only never changes resources. delete-approved-manual-snapshots deletes eligible approved manual snapshots.

  MinimumSnapshotAgeDays:
    Type: Number
    Default: 180
    Description: Manual snapshot must be at least this many days old before deletion consideration.

  RequireApprovalTag:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Snapshot deletion requires ApprovalTagKey=ApprovalTagValue when true.

  ApprovalTagKey:
    Type: String
    Default: CostOptApproved
    Description: Tag key required when RequireApprovalTag is true.

  ApprovalTagValue:
    Type: String
    Default: 'true'
    Description: Tag value required when RequireApprovalTag is true.

  IgnoreTagKeysCsv:
    Type: String
    Default: CostOptIgnore,DoNotDelete,DoNotModify,ComplianceHold,LegalHold,Production,prod
    Description: Comma-separated tag keys that prevent snapshot deletion.

  IgnoreTagPairsCsv:
    Type: String
    Default: CostOptIgnore=true,DoNotDelete=true,DoNotModify=true,ComplianceHold=true,LegalHold=true,Environment=Production,Environment=Prod
    Description: Comma-separated key=value tag pairs that prevent snapshot deletion.

  IncludeSnapshotNamePrefixesCsv:
    Type: String
    Default: ''
    Description: Optional comma-separated snapshot name prefixes to include. Leave blank to include all.

  ExcludeSnapshotNamePrefixesCsv:
    Type: String
    Default: ''
    Description: Optional comma-separated snapshot name prefixes to exclude.

  MaxSnapshotsPerRun:
    Type: Number
    Default: 50
    Description: Safety limit. 0 means no limit. Recommended actual deletion batch is 1 to 10.

  ContinueOnAccessDenied:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Continue scanning other Regions/resources when blocked by IAM/SCP.

  ScheduleEnabled:
    Type: String
    Default: 'false'
    AllowedValues: ['true', 'false']
    Description: Whether to create an enabled EventBridge schedule.

  ScheduleExpression:
    Type: String
    Default: rate(30 days)
    Description: EventBridge schedule expression used when ScheduleEnabled is true.

Conditions:
  IsScheduleEnabled: !Equals [!Ref ScheduleEnabled, 'true']

Resources:
  RemediatorRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: ElastiCacheSnapshotReviewPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Sid: DiscoverRegions
                Effect: Allow
                Action:
                  - ec2:DescribeRegions
                Resource: '*'
              - Sid: ReadElastiCache
                Effect: Allow
                Action:
                  - elasticache:DescribeCacheClusters
                  - elasticache:DescribeReplicationGroups
                  - elasticache:DescribeSnapshots
                  - elasticache:ListTagsForResource
                Resource: '*'
              - Sid: DeleteApprovedManualSnapshots
                Effect: Allow
                Action:
                  - elasticache:DeleteSnapshot
                Resource: '*'

  RemediatorFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Ref ProjectName
      Description: Purpose-specific Lambda for ElastiCache snapshot review and guarded manual snapshot cleanup.
      Runtime: python3.12
      Handler: index.handler
      Timeout: 900
      MemorySize: 512
      Role: !GetAtt RemediatorRole.Arn
      Environment:
        Variables:
          TARGET_REGIONS_CSV: !Ref TargetRegionsCsv
          EXCLUDED_REGIONS_CSV: !Ref ExcludedRegionsCsv
          DRY_RUN: !Ref DryRun
          ACTION_MODE: !Ref ActionMode
          MINIMUM_SNAPSHOT_AGE_DAYS: !Ref MinimumSnapshotAgeDays
          REQUIRE_APPROVAL_TAG: !Ref RequireApprovalTag
          APPROVAL_TAG_KEY: !Ref ApprovalTagKey
          APPROVAL_TAG_VALUE: !Ref ApprovalTagValue
          IGNORE_TAG_KEYS_CSV: !Ref IgnoreTagKeysCsv
          IGNORE_TAG_PAIRS_CSV: !Ref IgnoreTagPairsCsv
          INCLUDE_SNAPSHOT_NAME_PREFIXES_CSV: !Ref IncludeSnapshotNamePrefixesCsv
          EXCLUDE_SNAPSHOT_NAME_PREFIXES_CSV: !Ref ExcludeSnapshotNamePrefixesCsv
          MAX_SNAPSHOTS_PER_RUN: !Ref MaxSnapshotsPerRun
          CONTINUE_ON_ACCESS_DENIED: !Ref ContinueOnAccessDenied
      Code:
        ZipFile: |
          import json
          import os
          import traceback
          from datetime import datetime, timezone

          import boto3
          from botocore.exceptions import ClientError

          def utc_now():
              return datetime.now(timezone.utc)

          def utc_now_iso():
              return utc_now().isoformat()

          def env_bool(name, default=False):
              return os.environ.get(name, str(default)).strip().lower() in ('1', 'true', 'yes', 'y')

          def env_int(name, default=0):
              try:
                  return int(os.environ.get(name, str(default)).strip())
              except Exception:
                  return default

          def split_csv(value):
              if value is None:
                  return []
              return [x.strip() for x in str(value).split(',') if x.strip()]

          def parse_tag_pairs(value):
              pairs = {}
              for item in split_csv(value):
                  if '=' not in item:
                      continue
                  key, val = item.split('=', 1)
                  if key.strip():
                      pairs[key.strip()] = val.strip()
              return pairs

          def age_days(dt):
              if not dt:
                  return None
              if dt.tzinfo is None:
                  dt = dt.replace(tzinfo=timezone.utc)
              return (utc_now() - dt).days

          def get_enabled_regions(excluded_regions):
              requested = os.environ.get('TARGET_REGIONS_CSV', 'all-enabled').strip()
              if requested and requested.lower() != 'all-enabled':
                  regions = split_csv(requested)
              else:
                  ec2 = boto3.client('ec2', region_name=os.environ.get('AWS_REGION', 'us-east-1'))
                  response = ec2.describe_regions(AllRegions=False)
                  regions = sorted([r['RegionName'] for r in response.get('Regions', [])])
              return [r for r in regions if r not in excluded_regions]

          def tags_to_dict(tag_list):
              return {t.get('Key'): t.get('Value', '') for t in tag_list or [] if t.get('Key')}

          def has_ignore_tag(tags, ignore_keys, ignore_pairs):
              for key in ignore_keys:
                  if key in tags:
                      return True, 'ignore_tag_key=' + key
              for key, value in ignore_pairs.items():
                  if tags.get(key) == value:
                      return True, 'ignore_tag_pair=' + key + '=' + value
              return False, ''

          def has_approval_tag(tags, settings):
              if not settings['require_approval_tag']:
                  return True
              return tags.get(settings['approval_tag_key']) == settings['approval_tag_value']

          def name_included(name, include_prefixes):
              if not include_prefixes:
                  return True
              return any((name or '').startswith(prefix) for prefix in include_prefixes)

          def name_excluded(name, exclude_prefixes):
              if not exclude_prefixes:
                  return False
              return any((name or '').startswith(prefix) for prefix in exclude_prefixes)

          def safe_list_tags(client, arn):
              if not arn:
                  return {}, None
              try:
                  response = client.list_tags_for_resource(ResourceName=arn)
                  return tags_to_dict(response.get('TagList', [])), None
              except ClientError as e:
                  return {}, e

          def paginate(client, operation_name, result_key, **kwargs):
              items = []
              try:
                  paginator = client.get_paginator(operation_name)
                  for page in paginator.paginate(**kwargs):
                      items.extend(page.get(result_key, []))
                  return items
              except Exception:
                  marker_key = 'Marker'
                  next_key = 'Marker'
                  request = dict(kwargs)
                  while True:
                      response = getattr(client, operation_name)(**request)
                      items.extend(response.get(result_key, []))
                      marker = response.get(next_key)
                      if not marker:
                          break
                      request[marker_key] = marker
                  return items

          def list_clusters(client):
              return paginate(client, 'describe_cache_clusters', 'CacheClusters', ShowCacheNodeInfo=False)

          def list_replication_groups(client):
              return paginate(client, 'describe_replication_groups', 'ReplicationGroups')

          def list_manual_snapshots(client):
              try:
                  return paginate(client, 'describe_snapshots', 'Snapshots', SnapshotSource='manual')
              except ClientError as e:
                  # Some older/edge accounts may reject SnapshotSource; fallback and filter manually.
                  if e.response.get('Error', {}).get('Code') in ('InvalidParameterValue', 'InvalidParameterCombination'):
                      snaps = paginate(client, 'describe_snapshots', 'Snapshots')
                      return [s for s in snaps if str(s.get('SnapshotSource', '')).lower() == 'manual']
                  raise

          def snapshot_arn(snapshot):
              for key in ('ARN', 'SnapshotArn', 'SnapshotARN'):
                  if snapshot.get(key):
                      return snapshot.get(key)
              return None

          def scan_region(region, settings):
              client = boto3.client('elasticache', region_name=region)
              result = {
                  'region': region,
                  'evaluated_clusters': 0,
                  'cache_clusters': 0,
                  'replication_groups': 0,
                  'manual_snapshots': 0,
                  'old_manual_snapshots': 0,
                  'eligible': 0,
                  'would_delete_snapshots': 0,
                  'deleted_snapshots': 0,
                  'skipped_too_new': 0,
                  'skipped_include_prefix': 0,
                  'skipped_exclude_prefix': 0,
                  'skipped_missing_approval_tag': 0,
                  'skipped_ignore_tag': 0,
                  'tag_read_access_denied': 0,
                  'access_denied': 0,
                  'failed': 0,
                  'failures': [],
                  'clusters': [],
                  'replication_group_inventory': [],
                  'candidates': [],
                  'deleted_snapshot_names': []
              }

              try:
                  clusters = list_clusters(client)
                  replication_groups = list_replication_groups(client)
                  snapshots = list_manual_snapshots(client)
              except ClientError as e:
                  code = e.response.get('Error', {}).get('Code', 'Unknown')
                  message = e.response.get('Error', {}).get('Message', str(e))
                  result['failures'].append({'operation': 'DescribeElastiCacheResources', 'error_code': code, 'error_message': message})
                  if code in ('AccessDenied', 'AccessDeniedException') or 'explicit deny' in message.lower():
                      result['access_denied'] += 1
                  else:
                      result['failed'] += 1
                  if not settings['continue_on_access_denied']:
                      raise
                  return result

              result['evaluated_clusters'] = len(clusters)
              result['cache_clusters'] = len(clusters)
              result['replication_groups'] = len(replication_groups)
              result['manual_snapshots'] = len(snapshots)

              for c in clusters:
                  result['clusters'].append({
                      'cache_cluster_id': c.get('CacheClusterId'),
                      'engine': c.get('Engine'),
                      'engine_version': c.get('EngineVersion'),
                      'cache_node_type': c.get('CacheNodeType'),
                      'cache_cluster_status': c.get('CacheClusterStatus'),
                      'num_cache_nodes': c.get('NumCacheNodes'),
                      'replication_group_id': c.get('ReplicationGroupId'),
                      'preferred_availability_zone': c.get('PreferredAvailabilityZone'),
                      'snapshot_retention_limit': c.get('SnapshotRetentionLimit')
                  })

              for rg in replication_groups:
                  result['replication_group_inventory'].append({
                      'replication_group_id': rg.get('ReplicationGroupId'),
                      'description': rg.get('Description'),
                      'status': rg.get('Status'),
                      'engine': rg.get('Engine'),
                      'cache_node_type': rg.get('CacheNodeType'),
                      'member_clusters': rg.get('MemberClusters', []),
                      'automatic_failover': rg.get('AutomaticFailover'),
                      'multi_az': rg.get('MultiAZ'),
                      'cluster_enabled': rg.get('ClusterEnabled')
                  })

              for snap in snapshots:
                  if settings['max_snapshots_per_run'] and (result['would_delete_snapshots'] + result['deleted_snapshots']) >= settings['max_snapshots_per_run']:
                      result['stopped_due_to_max_snapshots_per_run'] = True
                      return result

                  name = snap.get('SnapshotName')
                  source = snap.get('SnapshotSource')
                  created = snap.get('NodeSnapshots', [{}])[0].get('SnapshotCreateTime') if snap.get('NodeSnapshots') else snap.get('SnapshotCreateTime')
                  snap_age = age_days(created)
                  arn = snapshot_arn(snap)
                  tags, tag_error = safe_list_tags(client, arn)

                  item = {
                      'snapshot_name': name,
                      'region': region,
                      'snapshot_source': source,
                      'snapshot_status': snap.get('SnapshotStatus'),
                      'engine': snap.get('Engine'),
                      'engine_version': snap.get('EngineVersion'),
                      'cache_cluster_id': snap.get('CacheClusterId'),
                      'replication_group_id': snap.get('ReplicationGroupId'),
                      'cache_node_type': snap.get('CacheNodeType'),
                      'num_node_snapshots': len(snap.get('NodeSnapshots', [])),
                      'created_time': str(created) if created else None,
                      'age_days': snap_age,
                      'arn': arn,
                      'tags': tags
                  }

                  if tag_error:
                      code = tag_error.response.get('Error', {}).get('Code', 'Unknown')
                      message = tag_error.response.get('Error', {}).get('Message', str(tag_error))
                      item['tag_read_error'] = {'error_code': code, 'error_message': message}
                      if code in ('AccessDenied', 'AccessDeniedException') or 'explicit deny' in message.lower():
                          result['tag_read_access_denied'] += 1
                      else:
                          result['failed'] += 1

                  if not name_included(name, settings['include_snapshot_prefixes']):
                      result['skipped_include_prefix'] += 1
                      continue

                  if name_excluded(name, settings['exclude_snapshot_prefixes']):
                      result['skipped_exclude_prefix'] += 1
                      item['action'] = 'skipped_exclude_prefix'
                      result['candidates'].append(item)
                      continue

                  if snap_age is None or snap_age < settings['minimum_snapshot_age_days']:
                      result['skipped_too_new'] += 1
                      item['action'] = 'skipped_too_new'
                      result['candidates'].append(item)
                      continue

                  result['old_manual_snapshots'] += 1

                  ignore, reason = has_ignore_tag(tags, settings['ignore_tag_keys'], settings['ignore_tag_pairs'])
                  if ignore:
                      result['skipped_ignore_tag'] += 1
                      item['action'] = 'skipped_ignore_tag'
                      item['reason'] = reason
                      result['candidates'].append(item)
                      continue

                  if not has_approval_tag(tags, settings):
                      result['skipped_missing_approval_tag'] += 1
                      item['action'] = 'skipped_missing_approval_tag'
                      item['required_tag'] = settings['approval_tag_key'] + '=' + settings['approval_tag_value']
                      result['candidates'].append(item)
                      continue

                  result['eligible'] += 1

                  if settings['action_mode'] != 'delete-approved-manual-snapshots':
                      item['action'] = 'review_only_candidate'
                      result['candidates'].append(item)
                      continue

                  item['action'] = 'would_delete_snapshot' if settings['dry_run'] else 'delete_snapshot'

                  if settings['dry_run']:
                      result['would_delete_snapshots'] += 1
                      result['candidates'].append(item)
                      continue

                  try:
                      client.delete_snapshot(SnapshotName=name)
                      result['deleted_snapshots'] += 1
                      result['deleted_snapshot_names'].append(name)
                  except ClientError as e:
                      code = e.response.get('Error', {}).get('Code', 'Unknown')
                      message = e.response.get('Error', {}).get('Message', str(e))
                      result['failures'].append({'snapshot_name': name, 'operation': 'DeleteSnapshot', 'error_code': code, 'error_message': message})
                      if code in ('AccessDenied', 'AccessDeniedException') or 'explicit deny' in message.lower():
                          result['access_denied'] += 1
                      else:
                          result['failed'] += 1
                      if not settings['continue_on_access_denied']:
                          raise

              return result

          def handler(event, context):
              settings = {
                  'excluded_regions': split_csv(os.environ.get('EXCLUDED_REGIONS_CSV', '')),
                  'dry_run': env_bool('DRY_RUN', True),
                  'action_mode': os.environ.get('ACTION_MODE', 'review-only'),
                  'minimum_snapshot_age_days': env_int('MINIMUM_SNAPSHOT_AGE_DAYS', 180),
                  'require_approval_tag': env_bool('REQUIRE_APPROVAL_TAG', True),
                  'approval_tag_key': os.environ.get('APPROVAL_TAG_KEY', 'CostOptApproved'),
                  'approval_tag_value': os.environ.get('APPROVAL_TAG_VALUE', 'true'),
                  'ignore_tag_keys': split_csv(os.environ.get('IGNORE_TAG_KEYS_CSV', '')),
                  'ignore_tag_pairs': parse_tag_pairs(os.environ.get('IGNORE_TAG_PAIRS_CSV', '')),
                  'include_snapshot_prefixes': split_csv(os.environ.get('INCLUDE_SNAPSHOT_NAME_PREFIXES_CSV', '')),
                  'exclude_snapshot_prefixes': split_csv(os.environ.get('EXCLUDE_SNAPSHOT_NAME_PREFIXES_CSV', '')),
                  'max_snapshots_per_run': env_int('MAX_SNAPSHOTS_PER_RUN', 50),
                  'continue_on_access_denied': env_bool('CONTINUE_ON_ACCESS_DENIED', True),
              }

              if isinstance(event, dict):
                  if 'dry_run' in event:
                      settings['dry_run'] = bool(event['dry_run'])
                  if 'action_mode' in event:
                      settings['action_mode'] = str(event['action_mode'])
                  if 'require_approval_tag' in event:
                      settings['require_approval_tag'] = bool(event['require_approval_tag'])
                  if 'minimum_snapshot_age_days' in event:
                      settings['minimum_snapshot_age_days'] = int(event['minimum_snapshot_age_days'])
                  if 'max_snapshots_per_run' in event:
                      settings['max_snapshots_per_run'] = int(event['max_snapshots_per_run'])
                  if 'target_regions' in event and event['target_regions']:
                      os.environ['TARGET_REGIONS_CSV'] = ','.join(event['target_regions'])
                  if 'excluded_regions' in event and event['excluded_regions']:
                      settings['excluded_regions'] = list(event['excluded_regions'])

              if settings['action_mode'] not in ('review-only', 'delete-approved-manual-snapshots'):
                  settings['action_mode'] = 'review-only'

              summary = {
                  'started_at': utc_now_iso(),
                  'dry_run': settings['dry_run'],
                  'action_mode': settings['action_mode'],
                  'minimum_snapshot_age_days': settings['minimum_snapshot_age_days'],
                  'require_approval_tag': settings['require_approval_tag'],
                  'approval_tag': settings['approval_tag_key'] + '=' + settings['approval_tag_value'],
                  'max_snapshots_per_run': settings['max_snapshots_per_run'],
                  'excluded_regions': settings['excluded_regions'],
                  'regions': [],
                  'totals': {
                      'evaluated_clusters': 0,
                      'cache_clusters': 0,
                      'replication_groups': 0,
                      'manual_snapshots': 0,
                      'old_manual_snapshots': 0,
                      'eligible': 0,
                      'would_delete_snapshots': 0,
                      'deleted_snapshots': 0,
                      'skipped_too_new': 0,
                      'skipped_include_prefix': 0,
                      'skipped_exclude_prefix': 0,
                      'skipped_missing_approval_tag': 0,
                      'skipped_ignore_tag': 0,
                      'tag_read_access_denied': 0,
                      'access_denied': 0,
                      'failed': 0
                  },
                  'region_results': []
              }

              try:
                  regions = get_enabled_regions(settings['excluded_regions'])
                  summary['regions'] = regions
                  for region in regions:
                      try:
                          rr = scan_region(region, settings)
                      except Exception as e:
                          rr = {
                              'region': region,
                              'evaluated_clusters': 0,
                              'cache_clusters': 0,
                              'replication_groups': 0,
                              'manual_snapshots': 0,
                              'old_manual_snapshots': 0,
                              'eligible': 0,
                              'would_delete_snapshots': 0,
                              'deleted_snapshots': 0,
                              'skipped_too_new': 0,
                              'skipped_include_prefix': 0,
                              'skipped_exclude_prefix': 0,
                              'skipped_missing_approval_tag': 0,
                              'skipped_ignore_tag': 0,
                              'tag_read_access_denied': 0,
                              'access_denied': 0,
                              'failed': 1,
                              'failures': [{'error_code': type(e).__name__, 'error_message': str(e), 'trace': traceback.format_exc()}],
                              'clusters': [],
                              'replication_group_inventory': [],
                              'candidates': [],
                              'deleted_snapshot_names': []
                          }
                      summary['region_results'].append(rr)
                      for key in summary['totals']:
                          summary['totals'][key] += int(rr.get(key, 0) or 0)
              finally:
                  summary['ended_at'] = utc_now_iso()
                  print(json.dumps(summary, indent=2, default=str))
              return summary

  RemediatorScheduleRule:
    Type: AWS::Events::Rule
    Condition: IsScheduleEnabled
    Properties:
      Name: !Sub '${ProjectName}-schedule'
      Description: Scheduled run for ElastiCache Snapshot Review Remediator.
      ScheduleExpression: !Ref ScheduleExpression
      State: ENABLED
      Targets:
        - Arn: !GetAtt RemediatorFunction.Arn
          Id: ElastiCacheSnapshotReviewTarget
          Input: '{"source":"eventbridge-schedule","dry_run":true,"action_mode":"review-only"}'

  RemediatorSchedulePermission:
    Type: AWS::Lambda::Permission
    Condition: IsScheduleEnabled
    Properties:
      FunctionName: !Ref RemediatorFunction
      Action: lambda:InvokeFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt RemediatorScheduleRule.Arn

Outputs:
  RemediatorFunctionName:
    Description: Lambda function name.
    Value: !Ref RemediatorFunction

  RemediatorFunctionArn:
    Description: Lambda function ARN.
    Value: !GetAtt RemediatorFunction.Arn

  DryRunSetting:
    Description: Current DryRun setting.
    Value: !Ref DryRun

  ActionModeSetting:
    Description: Current ActionMode setting.
    Value: !Ref ActionMode

  ManualReviewExample:
    Description: Example manual review invocation payload.
    Value: !Sub 'aws lambda invoke --function-name ${RemediatorFunction} --cli-binary-format raw-in-base64-out --payload ''{"dry_run":true,"action_mode":"review-only","require_approval_tag":false}'' response.json'
