AWSTemplateFormatVersion: '2010-09-09'
Description: >
  AWS Backup Recovery Point Review Remediator.
  Purpose-specific, opt-in remediation module for reviewing AWS Backup vaults and
  recovery points that may be old, large, or costly. Optional deletion is supported
  only with explicit approval tags and conservative safety checks.

Parameters:
  ProjectName:
    Type: String
    Default: backup-rp-review
    Description: Short base name used for created resources.

  TargetRegionsCsv:
    Type: String
    Default: all-enabled
    Description: Comma-separated AWS Regions to scan, or all-enabled to scan all enabled Regions.

  ExcludedRegionsCsv:
    Type: String
    Default: ap-northeast-3
    Description: Comma-separated AWS Regions to exclude from scanning/remediation.

  DryRun:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: When true, only report what would happen. When false, perform eligible actions.

  ActionMode:
    Type: String
    Default: review-only
    AllowedValues:
      - review-only
      - delete-approved-recovery-points
    Description: review-only never changes resources. delete-approved-recovery-points deletes eligible approved recovery points.

  MinimumRecoveryPointAgeDays:
    Type: Number
    Default: 180
    Description: Recovery points must be at least this many days old to be considered delete candidates.

  MinimumRecoveryPointSizeGiB:
    Type: Number
    Default: 0
    Description: Optional size filter. 0 means no minimum size filter.

  RequireApprovalTag:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Recovery point deletion requires ApprovalTagKey=ApprovalTagValue when true. Strongly recommended.

  ApprovalTagKey:
    Type: String
    Default: CostOptApproved
    Description: Tag key required when RequireApprovalTag is true.

  ApprovalTagValue:
    Type: String
    Default: 'true'
    Description: Tag value required when RequireApprovalTag is true.

  IgnoreTagKeysCsv:
    Type: String
    Default: CostOptIgnore,DoNotDelete,DoNotModify,ComplianceHold,LegalHold
    Description: Comma-separated tag keys that prevent deletion when present.

  IgnoreTagPairsCsv:
    Type: String
    Default: CostOptIgnore=true,DoNotDelete=true,DoNotModify=true,ComplianceHold=true,LegalHold=true
    Description: Comma-separated key=value tag pairs that prevent deletion.

  IncludeVaultNamePrefixesCsv:
    Type: String
    Default: ''
    Description: Optional comma-separated backup vault name prefixes to include. Leave blank to include all.

  ExcludeVaultNamePrefixesCsv:
    Type: String
    Default: ''
    Description: Optional comma-separated backup vault name prefixes to exclude.

  IncludeResourceTypesCsv:
    Type: String
    Default: ''
    Description: Optional comma-separated AWS Backup resource types to include, such as EC2,EBS,RDS. Leave blank to include all.

  ExcludeResourceTypesCsv:
    Type: String
    Default: ''
    Description: Optional comma-separated AWS Backup resource types to exclude.

  SkipContinuousRecoveryPoints:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Skip recovery points with recovery point type CONTINUOUS when true.

  SkipRecoveryPointsWithDeleteAt:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Skip recovery points that already have a lifecycle DeleteAt date.

  MaxRecoveryPointsPerRun:
    Type: Number
    Default: 100
    Description: Safety limit. 0 means no limit. Recommended actual delete batch is 1 to 10.

  ContinueOnAccessDenied:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Continue scanning other Regions/vaults/recovery points when blocked by IAM/SCP/vault policy.

  ScheduleEnabled:
    Type: String
    Default: 'false'
    AllowedValues: ['true', 'false']
    Description: Whether to create an enabled EventBridge schedule.

  ScheduleExpression:
    Type: String
    Default: rate(30 days)
    Description: EventBridge schedule expression used when ScheduleEnabled is true.

Conditions:
  IsScheduleEnabled: !Equals [!Ref ScheduleEnabled, 'true']

Resources:
  RemediatorRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: AwsBackupRecoveryPointReviewPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Sid: DiscoverRegions
                Effect: Allow
                Action:
                  - ec2:DescribeRegions
                Resource: '*'
              - Sid: ReadBackup
                Effect: Allow
                Action:
                  - backup:ListBackupVaults
                  - backup:ListRecoveryPointsByBackupVault
                  - backup:ListTags
                  - backup:DescribeRecoveryPoint
                Resource: '*'
              - Sid: DeleteApprovedRecoveryPoints
                Effect: Allow
                Action:
                  - backup:DeleteRecoveryPoint
                Resource: '*'

  RemediatorFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Ref ProjectName
      Description: Purpose-specific Lambda for AWS Backup recovery point review and guarded deletion.
      Runtime: python3.12
      Handler: index.handler
      Timeout: 900
      MemorySize: 512
      Role: !GetAtt RemediatorRole.Arn
      Environment:
        Variables:
          TARGET_REGIONS_CSV: !Ref TargetRegionsCsv
          EXCLUDED_REGIONS_CSV: !Ref ExcludedRegionsCsv
          DRY_RUN: !Ref DryRun
          ACTION_MODE: !Ref ActionMode
          MINIMUM_RECOVERY_POINT_AGE_DAYS: !Ref MinimumRecoveryPointAgeDays
          MINIMUM_RECOVERY_POINT_SIZE_GIB: !Ref MinimumRecoveryPointSizeGiB
          REQUIRE_APPROVAL_TAG: !Ref RequireApprovalTag
          APPROVAL_TAG_KEY: !Ref ApprovalTagKey
          APPROVAL_TAG_VALUE: !Ref ApprovalTagValue
          IGNORE_TAG_KEYS_CSV: !Ref IgnoreTagKeysCsv
          IGNORE_TAG_PAIRS_CSV: !Ref IgnoreTagPairsCsv
          INCLUDE_VAULT_NAME_PREFIXES_CSV: !Ref IncludeVaultNamePrefixesCsv
          EXCLUDE_VAULT_NAME_PREFIXES_CSV: !Ref ExcludeVaultNamePrefixesCsv
          INCLUDE_RESOURCE_TYPES_CSV: !Ref IncludeResourceTypesCsv
          EXCLUDE_RESOURCE_TYPES_CSV: !Ref ExcludeResourceTypesCsv
          SKIP_CONTINUOUS_RECOVERY_POINTS: !Ref SkipContinuousRecoveryPoints
          SKIP_RECOVERY_POINTS_WITH_DELETE_AT: !Ref SkipRecoveryPointsWithDeleteAt
          MAX_RECOVERY_POINTS_PER_RUN: !Ref MaxRecoveryPointsPerRun
          CONTINUE_ON_ACCESS_DENIED: !Ref ContinueOnAccessDenied
      Code:
        ZipFile: |
          import json
          import os
          import traceback
          from datetime import datetime, timezone

          import boto3
          from botocore.exceptions import ClientError, EndpointConnectionError, ConnectTimeoutError

          GIB = 1024 ** 3

          def utc_now():
              return datetime.now(timezone.utc)

          def utc_now_iso():
              return utc_now().isoformat()

          def env_bool(name, default=False):
              return os.environ.get(name, str(default)).strip().lower() in ('1', 'true', 'yes', 'y')

          def env_int(name, default=0):
              try:
                  return int(os.environ.get(name, str(default)).strip())
              except Exception:
                  return default

          def split_csv(value):
              if value is None:
                  return []
              return [x.strip() for x in str(value).split(',') if x.strip()]

          def parse_tag_pairs(value):
              pairs = {}
              for item in split_csv(value):
                  if '=' not in item:
                      continue
                  key, val = item.split('=', 1)
                  if key.strip():
                      pairs[key.strip()] = val.strip()
              return pairs

          def get_enabled_regions(excluded_regions):
              requested = os.environ.get('TARGET_REGIONS_CSV', 'all-enabled').strip()
              if requested and requested.lower() != 'all-enabled':
                  regions = split_csv(requested)
              else:
                  ec2 = boto3.client('ec2', region_name=os.environ.get('AWS_REGION', 'us-east-1'))
                  response = ec2.describe_regions(AllRegions=False)
                  regions = sorted([r['RegionName'] for r in response.get('Regions', [])])
              return [r for r in regions if r not in excluded_regions]

          def name_included(name, include_prefixes):
              if not include_prefixes:
                  return True
              return any((name or '').startswith(prefix) for prefix in include_prefixes)

          def name_excluded(name, exclude_prefixes):
              if not exclude_prefixes:
                  return False
              return any((name or '').startswith(prefix) for prefix in exclude_prefixes)

          def resource_type_included(resource_type, include_types):
              if not include_types:
                  return True
              return (resource_type or '') in include_types

          def resource_type_excluded(resource_type, exclude_types):
              if not exclude_types:
                  return False
              return (resource_type or '') in exclude_types

          def has_ignore_tag(tags, ignore_keys, ignore_pairs):
              for key in ignore_keys:
                  if key in tags:
                      return True, 'ignore_tag_key=' + key
              for key, value in ignore_pairs.items():
                  if tags.get(key) == value:
                      return True, 'ignore_tag_pair=' + key + '=' + value
              return False, ''

          def has_approval_tag(tags, settings):
              if not settings['require_approval_tag']:
                  return True
              return tags.get(settings['approval_tag_key']) == settings['approval_tag_value']

          def age_days(dt):
              if not dt:
                  return None
              if dt.tzinfo is None:
                  dt = dt.replace(tzinfo=timezone.utc)
              return (utc_now() - dt).days

          def bytes_to_gib(value):
              try:
                  return round(float(value or 0) / GIB, 3)
              except Exception:
                  return 0.0

          def get_tags(backup, arn):
              if not arn:
                  return {}
              try:
                  response = backup.list_tags(ResourceArn=arn)
                  return response.get('Tags', {}) or {}
              except ClientError:
                  return {}

          def list_vaults(backup):
              vaults = []
              paginator = backup.get_paginator('list_backup_vaults')
              for page in paginator.paginate():
                  vaults.extend(page.get('BackupVaultList', []))
              return vaults

          def list_recovery_points(backup, vault_name):
              points = []
              paginator = backup.get_paginator('list_recovery_points_by_backup_vault')
              for page in paginator.paginate(BackupVaultName=vault_name):
                  points.extend(page.get('RecoveryPoints', []))
              return points

          def delete_recovery_point(backup, vault_name, recovery_point_arn):
              backup.delete_recovery_point(
                  BackupVaultName=vault_name,
                  RecoveryPointArn=recovery_point_arn
              )

          def scan_region(region, settings):
              backup = boto3.client('backup', region_name=region)
              result = {
                  'region': region,
                  'evaluated_vaults': 0,
                  'evaluated_recovery_points': 0,
                  'old_recovery_points': 0,
                  'large_recovery_points': 0,
                  'eligible': 0,
                  'would_delete': 0,
                  'deleted': 0,
                  'total_backup_size_gib': 0.0,
                  'old_backup_size_gib': 0.0,
                  'eligible_backup_size_gib': 0.0,
                  'skipped_vault_include_prefix': 0,
                  'skipped_vault_exclude_prefix': 0,
                  'skipped_resource_type_include': 0,
                  'skipped_resource_type_exclude': 0,
                  'skipped_too_new': 0,
                  'skipped_too_small': 0,
                  'skipped_not_completed': 0,
                  'skipped_continuous': 0,
                  'skipped_has_delete_at': 0,
                  'skipped_missing_approval_tag': 0,
                  'skipped_ignore_tag': 0,
                  'access_denied': 0,
                  'failed': 0,
                  'failures': [],
                  'candidates': [],
                  'deleted_recovery_points': []
              }

              try:
                  vaults = list_vaults(backup)
              except (EndpointConnectionError, ConnectTimeoutError) as e:
                  result['failed'] += 1
                  result['failures'].append({'operation': 'ListBackupVaults', 'error_code': type(e).__name__, 'error_message': str(e)})
                  return result
              except ClientError as e:
                  code = e.response.get('Error', {}).get('Code', 'Unknown')
                  message = e.response.get('Error', {}).get('Message', str(e))
                  result['failures'].append({'operation': 'ListBackupVaults', 'error_code': code, 'error_message': message})
                  if code in ('AccessDeniedException', 'AccessDenied') or 'explicit deny' in message.lower():
                      result['access_denied'] += 1
                  else:
                      result['failed'] += 1
                  if not settings['continue_on_access_denied']:
                      raise
                  return result

              for vault in vaults:
                  vault_name = vault.get('BackupVaultName')
                  if not name_included(vault_name, settings['include_vault_name_prefixes']):
                      result['skipped_vault_include_prefix'] += 1
                      continue
                  if name_excluded(vault_name, settings['exclude_vault_name_prefixes']):
                      result['skipped_vault_exclude_prefix'] += 1
                      continue

                  result['evaluated_vaults'] += 1

                  try:
                      recovery_points = list_recovery_points(backup, vault_name)
                  except ClientError as e:
                      code = e.response.get('Error', {}).get('Code', 'Unknown')
                      message = e.response.get('Error', {}).get('Message', str(e))
                      result['failures'].append({'vault_name': vault_name, 'operation': 'ListRecoveryPointsByBackupVault', 'error_code': code, 'error_message': message})
                      if code in ('AccessDeniedException', 'AccessDenied') or 'explicit deny' in message.lower():
                          result['access_denied'] += 1
                      else:
                          result['failed'] += 1
                      if not settings['continue_on_access_denied']:
                          raise
                      continue

                  for rp in recovery_points:
                      if settings['max_recovery_points_per_run'] and (result['would_delete'] + result['deleted']) >= settings['max_recovery_points_per_run']:
                          result['stopped_due_to_max_recovery_points_per_run'] = True
                          return result

                      result['evaluated_recovery_points'] += 1
                      recovery_point_arn = rp.get('RecoveryPointArn')
                      resource_type = rp.get('ResourceType')
                      status = rp.get('Status')
                      creation_date = rp.get('CreationDate')
                      completion_date = rp.get('CompletionDate')
                      age = age_days(creation_date)
                      backup_size_bytes = int(rp.get('BackupSizeInBytes') or 0)
                      backup_size_gib = bytes_to_gib(backup_size_bytes)
                      result['total_backup_size_gib'] += backup_size_gib

                      lifecycle = rp.get('CalculatedLifecycle') or {}
                      delete_at = lifecycle.get('DeleteAt')
                      move_to_cold_at = lifecycle.get('MoveToColdStorageAt')
                      recovery_point_type = rp.get('RecoveryPointType')

                      item = {
                          'vault_name': vault_name,
                          'recovery_point_arn': recovery_point_arn,
                          'resource_arn': rp.get('ResourceArn'),
                          'resource_type': resource_type,
                          'status': status,
                          'recovery_point_type': recovery_point_type,
                          'creation_date': str(creation_date),
                          'completion_date': str(completion_date),
                          'age_days': age,
                          'backup_size_gib': backup_size_gib,
                          'delete_at': str(delete_at) if delete_at else None,
                          'move_to_cold_storage_at': str(move_to_cold_at) if move_to_cold_at else None
                      }

                      if not resource_type_included(resource_type, settings['include_resource_types']):
                          result['skipped_resource_type_include'] += 1
                          continue
                      if resource_type_excluded(resource_type, settings['exclude_resource_types']):
                          result['skipped_resource_type_exclude'] += 1
                          continue

                      if status != 'COMPLETED':
                          result['skipped_not_completed'] += 1
                          item['action'] = 'skipped_not_completed'
                          result['candidates'].append(item)
                          continue

                      if settings['skip_continuous_recovery_points'] and recovery_point_type == 'CONTINUOUS':
                          result['skipped_continuous'] += 1
                          item['action'] = 'skipped_continuous'
                          result['candidates'].append(item)
                          continue

                      if settings['skip_recovery_points_with_delete_at'] and delete_at:
                          result['skipped_has_delete_at'] += 1
                          item['action'] = 'skipped_has_delete_at'
                          result['candidates'].append(item)
                          continue

                      if age is None or age < settings['minimum_recovery_point_age_days']:
                          result['skipped_too_new'] += 1
                          item['action'] = 'skipped_too_new'
                          result['candidates'].append(item)
                          continue

                      result['old_recovery_points'] += 1
                      result['old_backup_size_gib'] += backup_size_gib

                      if settings['minimum_recovery_point_size_gib'] and backup_size_gib < settings['minimum_recovery_point_size_gib']:
                          result['skipped_too_small'] += 1
                          item['action'] = 'skipped_too_small'
                          result['candidates'].append(item)
                          continue

                      if settings['minimum_recovery_point_size_gib'] and backup_size_gib >= settings['minimum_recovery_point_size_gib']:
                          result['large_recovery_points'] += 1

                      tags = get_tags(backup, recovery_point_arn)
                      item['tags'] = tags

                      ignore, reason = has_ignore_tag(tags, settings['ignore_tag_keys'], settings['ignore_tag_pairs'])
                      if ignore:
                          result['skipped_ignore_tag'] += 1
                          item['action'] = 'skipped_ignore_tag'
                          item['reason'] = reason
                          result['candidates'].append(item)
                          continue

                      if not has_approval_tag(tags, settings):
                          result['skipped_missing_approval_tag'] += 1
                          item['action'] = 'skipped_missing_approval_tag'
                          item['required_tag'] = settings['approval_tag_key'] + '=' + settings['approval_tag_value']
                          result['candidates'].append(item)
                          continue

                      result['eligible'] += 1
                      result['eligible_backup_size_gib'] += backup_size_gib

                      if settings['action_mode'] != 'delete-approved-recovery-points':
                          item['action'] = 'review_only_candidate'
                          result['candidates'].append(item)
                          continue

                      item['action'] = 'would_delete' if settings['dry_run'] else 'delete'

                      if settings['dry_run']:
                          result['would_delete'] += 1
                          result['candidates'].append(item)
                          continue

                      try:
                          delete_recovery_point(backup, vault_name, recovery_point_arn)
                          result['deleted'] += 1
                          result['deleted_recovery_points'].append(item)
                      except ClientError as e:
                          code = e.response.get('Error', {}).get('Code', 'Unknown')
                          message = e.response.get('Error', {}).get('Message', str(e))
                          result['failures'].append({'vault_name': vault_name, 'recovery_point_arn': recovery_point_arn, 'operation': 'DeleteRecoveryPoint', 'error_code': code, 'error_message': message})
                          if code in ('AccessDeniedException', 'AccessDenied') or 'explicit deny' in message.lower():
                              result['access_denied'] += 1
                          else:
                              result['failed'] += 1
                          if not settings['continue_on_access_denied']:
                              raise

              result['total_backup_size_gib'] = round(result['total_backup_size_gib'], 3)
              result['old_backup_size_gib'] = round(result['old_backup_size_gib'], 3)
              result['eligible_backup_size_gib'] = round(result['eligible_backup_size_gib'], 3)
              return result

          def handler(event, context):
              settings = {
                  'excluded_regions': split_csv(os.environ.get('EXCLUDED_REGIONS_CSV', '')),
                  'dry_run': env_bool('DRY_RUN', True),
                  'action_mode': os.environ.get('ACTION_MODE', 'review-only'),
                  'minimum_recovery_point_age_days': env_int('MINIMUM_RECOVERY_POINT_AGE_DAYS', 180),
                  'minimum_recovery_point_size_gib': env_int('MINIMUM_RECOVERY_POINT_SIZE_GIB', 0),
                  'require_approval_tag': env_bool('REQUIRE_APPROVAL_TAG', True),
                  'approval_tag_key': os.environ.get('APPROVAL_TAG_KEY', 'CostOptApproved'),
                  'approval_tag_value': os.environ.get('APPROVAL_TAG_VALUE', 'true'),
                  'ignore_tag_keys': split_csv(os.environ.get('IGNORE_TAG_KEYS_CSV', '')),
                  'ignore_tag_pairs': parse_tag_pairs(os.environ.get('IGNORE_TAG_PAIRS_CSV', '')),
                  'include_vault_name_prefixes': split_csv(os.environ.get('INCLUDE_VAULT_NAME_PREFIXES_CSV', '')),
                  'exclude_vault_name_prefixes': split_csv(os.environ.get('EXCLUDE_VAULT_NAME_PREFIXES_CSV', '')),
                  'include_resource_types': split_csv(os.environ.get('INCLUDE_RESOURCE_TYPES_CSV', '')),
                  'exclude_resource_types': split_csv(os.environ.get('EXCLUDE_RESOURCE_TYPES_CSV', '')),
                  'skip_continuous_recovery_points': env_bool('SKIP_CONTINUOUS_RECOVERY_POINTS', True),
                  'skip_recovery_points_with_delete_at': env_bool('SKIP_RECOVERY_POINTS_WITH_DELETE_AT', True),
                  'max_recovery_points_per_run': env_int('MAX_RECOVERY_POINTS_PER_RUN', 100),
                  'continue_on_access_denied': env_bool('CONTINUE_ON_ACCESS_DENIED', True),
              }

              if isinstance(event, dict):
                  if 'dry_run' in event:
                      settings['dry_run'] = bool(event['dry_run'])
                  if 'action_mode' in event:
                      settings['action_mode'] = str(event['action_mode'])
                  if 'minimum_recovery_point_age_days' in event:
                      settings['minimum_recovery_point_age_days'] = int(event['minimum_recovery_point_age_days'])
                  if 'minimum_recovery_point_size_gib' in event:
                      settings['minimum_recovery_point_size_gib'] = int(event['minimum_recovery_point_size_gib'])
                  if 'require_approval_tag' in event:
                      settings['require_approval_tag'] = bool(event['require_approval_tag'])
                  if 'include_resource_types' in event:
                      settings['include_resource_types'] = list(event['include_resource_types'])
                  if 'exclude_resource_types' in event:
                      settings['exclude_resource_types'] = list(event['exclude_resource_types'])
                  if 'target_regions' in event and event['target_regions']:
                      os.environ['TARGET_REGIONS_CSV'] = ','.join(event['target_regions'])
                  if 'excluded_regions' in event and event['excluded_regions']:
                      settings['excluded_regions'] = list(event['excluded_regions'])

              if settings['action_mode'] not in ('review-only', 'delete-approved-recovery-points'):
                  settings['action_mode'] = 'review-only'

              summary = {
                  'started_at': utc_now_iso(),
                  'dry_run': settings['dry_run'],
                  'action_mode': settings['action_mode'],
                  'minimum_recovery_point_age_days': settings['minimum_recovery_point_age_days'],
                  'minimum_recovery_point_size_gib': settings['minimum_recovery_point_size_gib'],
                  'require_approval_tag': settings['require_approval_tag'],
                  'approval_tag': settings['approval_tag_key'] + '=' + settings['approval_tag_value'],
                  'skip_continuous_recovery_points': settings['skip_continuous_recovery_points'],
                  'skip_recovery_points_with_delete_at': settings['skip_recovery_points_with_delete_at'],
                  'include_vault_name_prefixes': settings['include_vault_name_prefixes'],
                  'exclude_vault_name_prefixes': settings['exclude_vault_name_prefixes'],
                  'include_resource_types': settings['include_resource_types'],
                  'exclude_resource_types': settings['exclude_resource_types'],
                  'max_recovery_points_per_run': settings['max_recovery_points_per_run'],
                  'excluded_regions': settings['excluded_regions'],
                  'regions': [],
                  'totals': {
                      'evaluated_vaults': 0,
                      'evaluated_recovery_points': 0,
                      'old_recovery_points': 0,
                      'large_recovery_points': 0,
                      'eligible': 0,
                      'would_delete': 0,
                      'deleted': 0,
                      'total_backup_size_gib': 0.0,
                      'old_backup_size_gib': 0.0,
                      'eligible_backup_size_gib': 0.0,
                      'skipped_vault_include_prefix': 0,
                      'skipped_vault_exclude_prefix': 0,
                      'skipped_resource_type_include': 0,
                      'skipped_resource_type_exclude': 0,
                      'skipped_too_new': 0,
                      'skipped_too_small': 0,
                      'skipped_not_completed': 0,
                      'skipped_continuous': 0,
                      'skipped_has_delete_at': 0,
                      'skipped_missing_approval_tag': 0,
                      'skipped_ignore_tag': 0,
                      'access_denied': 0,
                      'failed': 0
                  },
                  'region_results': []
              }

              try:
                  regions = get_enabled_regions(settings['excluded_regions'])
                  summary['regions'] = regions
                  for region in regions:
                      try:
                          rr = scan_region(region, settings)
                      except Exception as e:
                          rr = {
                              'region': region,
                              'evaluated_vaults': 0,
                              'evaluated_recovery_points': 0,
                              'old_recovery_points': 0,
                              'large_recovery_points': 0,
                              'eligible': 0,
                              'would_delete': 0,
                              'deleted': 0,
                              'total_backup_size_gib': 0.0,
                              'old_backup_size_gib': 0.0,
                              'eligible_backup_size_gib': 0.0,
                              'skipped_vault_include_prefix': 0,
                              'skipped_vault_exclude_prefix': 0,
                              'skipped_resource_type_include': 0,
                              'skipped_resource_type_exclude': 0,
                              'skipped_too_new': 0,
                              'skipped_too_small': 0,
                              'skipped_not_completed': 0,
                              'skipped_continuous': 0,
                              'skipped_has_delete_at': 0,
                              'skipped_missing_approval_tag': 0,
                              'skipped_ignore_tag': 0,
                              'access_denied': 0,
                              'failed': 1,
                              'failures': [{'error_code': type(e).__name__, 'error_message': str(e), 'trace': traceback.format_exc()}],
                              'candidates': [],
                              'deleted_recovery_points': []
                          }
                      summary['region_results'].append(rr)
                      for key in summary['totals']:
                          summary['totals'][key] += rr.get(key, 0) or 0
                  summary['totals']['total_backup_size_gib'] = round(summary['totals']['total_backup_size_gib'], 3)
                  summary['totals']['old_backup_size_gib'] = round(summary['totals']['old_backup_size_gib'], 3)
                  summary['totals']['eligible_backup_size_gib'] = round(summary['totals']['eligible_backup_size_gib'], 3)
              finally:
                  summary['ended_at'] = utc_now_iso()
                  print(json.dumps(summary, indent=2, default=str))
              return summary

  RemediatorScheduleRule:
    Type: AWS::Events::Rule
    Condition: IsScheduleEnabled
    Properties:
      Name: !Sub '${ProjectName}-schedule'
      Description: Scheduled run for AWS Backup Recovery Point Review Remediator.
      ScheduleExpression: !Ref ScheduleExpression
      State: ENABLED
      Targets:
        - Arn: !GetAtt RemediatorFunction.Arn
          Id: AwsBackupRecoveryPointReviewTarget
          Input: '{"source":"eventbridge-schedule","dry_run":true,"action_mode":"review-only"}'

  RemediatorSchedulePermission:
    Type: AWS::Lambda::Permission
    Condition: IsScheduleEnabled
    Properties:
      FunctionName: !Ref RemediatorFunction
      Action: lambda:InvokeFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt RemediatorScheduleRule.Arn

Outputs:
  RemediatorFunctionName:
    Description: Lambda function name.
    Value: !Ref RemediatorFunction

  RemediatorFunctionArn:
    Description: Lambda function ARN.
    Value: !GetAtt RemediatorFunction.Arn

  DryRunSetting:
    Description: Current DryRun setting.
    Value: !Ref DryRun

  ActionModeSetting:
    Description: Current ActionMode setting.
    Value: !Ref ActionMode

  ManualReviewExample:
    Description: Example manual review invocation payload.
    Value: !Sub 'aws lambda invoke --function-name ${RemediatorFunction} --cli-binary-format raw-in-base64-out --payload ''{"dry_run":true,"action_mode":"review-only","require_approval_tag":false}'' response.json'
