AWSTemplateFormatVersion: '2010-09-09'
Description: >
  FSx File System and Backup Review Remediator.
  Purpose-specific, opt-in remediation module for reviewing Amazon FSx file systems
  and backups. Optional remediation deletes explicitly approved old backups only.
  It does not delete FSx file systems.

Parameters:
  ProjectName:
    Type: String
    Default: fsx-backup-review
    Description: Short base name used for created resources.

  TargetRegionsCsv:
    Type: String
    Default: all-enabled
    Description: Comma-separated AWS Regions to scan, or all-enabled to scan all enabled Regions.

  ExcludedRegionsCsv:
    Type: String
    Default: ap-northeast-3
    Description: Comma-separated AWS Regions to exclude from scanning/remediation.

  DryRun:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: When true, only report what would happen. When false, perform eligible approved backup deletion.

  ActionMode:
    Type: String
    Default: review-only
    AllowedValues:
      - review-only
      - delete-approved-old-backups
    Description: review-only never changes resources. delete-approved-old-backups deletes eligible approved FSx backups.

  MinimumBackupAgeDays:
    Type: Number
    Default: 180
    Description: Backup must be at least this many days old before deletion consideration.

  RequireApprovalTag:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Backup deletion requires ApprovalTagKey=ApprovalTagValue when true.

  ApprovalTagKey:
    Type: String
    Default: CostOptApproved
    Description: Tag key required when RequireApprovalTag is true.

  ApprovalTagValue:
    Type: String
    Default: 'true'
    Description: Tag value required when RequireApprovalTag is true.

  IgnoreTagKeysCsv:
    Type: String
    Default: CostOptIgnore,DoNotDelete,DoNotModify,ComplianceHold,LegalHold,Production,prod
    Description: Comma-separated tag keys that prevent backup deletion.

  IgnoreTagPairsCsv:
    Type: String
    Default: CostOptIgnore=true,DoNotDelete=true,DoNotModify=true,ComplianceHold=true,LegalHold=true,Environment=Production,Environment=Prod
    Description: Comma-separated key=value tag pairs that prevent backup deletion.

  IncludeBackupIdPrefixesCsv:
    Type: String
    Default: ''
    Description: Optional comma-separated backup ID prefixes to include. Leave blank to include all.

  ExcludeBackupIdPrefixesCsv:
    Type: String
    Default: ''
    Description: Optional comma-separated backup ID prefixes to exclude.

  SkipAwsBackupManagedBackups:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Skip backups that appear to be managed by AWS Backup.

  MaxBackupsPerRun:
    Type: Number
    Default: 50
    Description: Safety limit. 0 means no limit. Recommended actual deletion batch is 1 to 10.

  ContinueOnAccessDenied:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Continue scanning other Regions/resources when blocked by IAM/SCP.

  ScheduleEnabled:
    Type: String
    Default: 'false'
    AllowedValues: ['true', 'false']
    Description: Whether to create an enabled EventBridge schedule.

  ScheduleExpression:
    Type: String
    Default: rate(30 days)
    Description: EventBridge schedule expression used when ScheduleEnabled is true.

Conditions:
  IsScheduleEnabled: !Equals [!Ref ScheduleEnabled, 'true']

Resources:
  RemediatorRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: FsxBackupReviewPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Sid: DiscoverRegions
                Effect: Allow
                Action:
                  - ec2:DescribeRegions
                Resource: '*'
              - Sid: ReadFSx
                Effect: Allow
                Action:
                  - fsx:DescribeFileSystems
                  - fsx:DescribeBackups
                  - fsx:ListTagsForResource
                Resource: '*'
              - Sid: DeleteApprovedBackups
                Effect: Allow
                Action:
                  - fsx:DeleteBackup
                Resource: '*'

  RemediatorFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Ref ProjectName
      Description: Purpose-specific Lambda for FSx file system and backup review with guarded approved backup cleanup.
      Runtime: python3.12
      Handler: index.handler
      Timeout: 900
      MemorySize: 512
      Role: !GetAtt RemediatorRole.Arn
      Environment:
        Variables:
          TARGET_REGIONS_CSV: !Ref TargetRegionsCsv
          EXCLUDED_REGIONS_CSV: !Ref ExcludedRegionsCsv
          DRY_RUN: !Ref DryRun
          ACTION_MODE: !Ref ActionMode
          MINIMUM_BACKUP_AGE_DAYS: !Ref MinimumBackupAgeDays
          REQUIRE_APPROVAL_TAG: !Ref RequireApprovalTag
          APPROVAL_TAG_KEY: !Ref ApprovalTagKey
          APPROVAL_TAG_VALUE: !Ref ApprovalTagValue
          IGNORE_TAG_KEYS_CSV: !Ref IgnoreTagKeysCsv
          IGNORE_TAG_PAIRS_CSV: !Ref IgnoreTagPairsCsv
          INCLUDE_BACKUP_ID_PREFIXES_CSV: !Ref IncludeBackupIdPrefixesCsv
          EXCLUDE_BACKUP_ID_PREFIXES_CSV: !Ref ExcludeBackupIdPrefixesCsv
          SKIP_AWS_BACKUP_MANAGED_BACKUPS: !Ref SkipAwsBackupManagedBackups
          MAX_BACKUPS_PER_RUN: !Ref MaxBackupsPerRun
          CONTINUE_ON_ACCESS_DENIED: !Ref ContinueOnAccessDenied
      Code:
        ZipFile: |
          import json
          import os
          import traceback
          from datetime import datetime, timezone

          import boto3
          from botocore.exceptions import ClientError

          def now():
              return datetime.now(timezone.utc)

          def split_csv(value):
              return [x.strip() for x in str(value or '').split(',') if x.strip()]

          def env_bool(name, default=False):
              return os.environ.get(name, str(default)).strip().lower() in ('1', 'true', 'yes', 'y')

          def env_int(name, default=0):
              try:
                  return int(os.environ.get(name, str(default)).strip())
              except Exception:
                  return default

          def parse_tag_pairs(value):
              pairs = {}
              for item in split_csv(value):
                  if '=' not in item:
                      continue
                  key, val = item.split('=', 1)
                  if key.strip():
                      pairs[key.strip()] = val.strip()
              return pairs

          def age_days(dt):
              if not dt:
                  return None
              if dt.tzinfo is None:
                  dt = dt.replace(tzinfo=timezone.utc)
              return (now() - dt).days

          def enabled_regions(excluded):
              requested = os.environ.get('TARGET_REGIONS_CSV', 'all-enabled').strip()
              if requested and requested.lower() != 'all-enabled':
                  regions = split_csv(requested)
              else:
                  ec2 = boto3.client('ec2', region_name=os.environ.get('AWS_REGION', 'us-east-1'))
                  response = ec2.describe_regions(AllRegions=False)
                  regions = sorted([r['RegionName'] for r in response.get('Regions', [])])
              return [r for r in regions if r not in excluded]

          def paginate(client, operation_name, result_key, **kwargs):
              items = []
              try:
                  paginator = client.get_paginator(operation_name)
                  for page in paginator.paginate(**kwargs):
                      items.extend(page.get(result_key, []))
                  return items
              except Exception:
                  request = dict(kwargs)
                  while True:
                      response = getattr(client, operation_name)(**request)
                      items.extend(response.get(result_key, []))
                      token = response.get('NextToken')
                      if not token:
                          break
                      request['NextToken'] = token
                  return items

          def tags_to_dict(tag_list):
              return {t.get('Key'): t.get('Value', '') for t in tag_list or [] if t.get('Key')}

          def safe_tags(client, arn):
              if not arn:
                  return {}, None
              try:
                  response = client.list_tags_for_resource(ResourceARN=arn)
                  return tags_to_dict(response.get('Tags', [])), None
              except ClientError as e:
                  return {}, e

          def has_ignore_tag(tags, ignore_keys, ignore_pairs):
              for key in ignore_keys:
                  if key in tags:
                      return True, 'ignore_tag_key=' + key
              for key, value in ignore_pairs.items():
                  if tags.get(key) == value:
                      return True, 'ignore_tag_pair=' + key + '=' + value
              return False, ''

          def has_approval_tag(tags, settings):
              if not settings['require_approval_tag']:
                  return True
              return tags.get(settings['approval_tag_key']) == settings['approval_tag_value']

          def id_included(value, prefixes):
              if not prefixes:
                  return True
              return any((value or '').startswith(prefix) for prefix in prefixes)

          def id_excluded(value, prefixes):
              if not prefixes:
                  return False
              return any((value or '').startswith(prefix) for prefix in prefixes)

          def appears_aws_backup_managed(backup, tags):
              name = str(backup.get('Name') or '').lower()
              lifecycle = str(backup.get('Lifecycle') or '').lower()
              if name.startswith('awsbackup') or 'awsbackup' in name:
                  return True
              if lifecycle == 'available' and backup.get('Type') == 'AWS_BACKUP':
                  return True
              aws_backup_keys = {'aws:backup:source-resource', 'aws:backup:backup-vault', 'aws:backup:job-id'}
              if any(k in tags for k in aws_backup_keys):
                  return True
              if str(tags.get('aws:backup:source-resource', '')).strip():
                  return True
              return False

          def fs_inventory(fs):
              windows = fs.get('WindowsConfiguration', {}) or {}
              lustre = fs.get('LustreConfiguration', {}) or {}
              ontap = fs.get('OntapConfiguration', {}) or {}
              openzfs = fs.get('OpenZFSConfiguration', {}) or {}

              return {
                  'file_system_id': fs.get('FileSystemId'),
                  'file_system_type': fs.get('FileSystemType'),
                  'lifecycle': fs.get('Lifecycle'),
                  'storage_capacity_gib': fs.get('StorageCapacity'),
                  'storage_type': fs.get('StorageType'),
                  'vpc_id': fs.get('VpcId'),
                  'subnet_ids': fs.get('SubnetIds', []),
                  'kms_key_id': fs.get('KmsKeyId'),
                  'creation_time': str(fs.get('CreationTime')) if fs.get('CreationTime') else None,
                  'dns_name': fs.get('DNSName'),
                  'windows_throughput_capacity': windows.get('ThroughputCapacity'),
                  'windows_deployment_type': windows.get('DeploymentType'),
                  'windows_automatic_backup_retention_days': windows.get('AutomaticBackupRetentionDays'),
                  'lustre_deployment_type': lustre.get('DeploymentType'),
                  'lustre_per_unit_storage_throughput': lustre.get('PerUnitStorageThroughput'),
                  'ontap_deployment_type': ontap.get('DeploymentType'),
                  'ontap_throughput_capacity': ontap.get('ThroughputCapacity'),
                  'openzfs_deployment_type': openzfs.get('DeploymentType'),
                  'openzfs_throughput_capacity': openzfs.get('ThroughputCapacity')
              }

          def scan_region(region, settings):
              client = boto3.client('fsx', region_name=region)
              result = {
                  'region': region,
                  'file_systems': 0,
                  'backups': 0,
                  'old_backups': 0,
                  'aws_backup_managed_backups': 0,
                  'eligible': 0,
                  'would_delete_backups': 0,
                  'deleted_backups': 0,
                  'skipped_too_new': 0,
                  'skipped_aws_backup_managed': 0,
                  'skipped_include_prefix': 0,
                  'skipped_exclude_prefix': 0,
                  'skipped_missing_approval_tag': 0,
                  'skipped_ignore_tag': 0,
                  'tag_read_access_denied': 0,
                  'access_denied': 0,
                  'failed': 0,
                  'failures': [],
                  'file_system_inventory': [],
                  'candidates': [],
                  'deleted_backup_ids': []
              }

              try:
                  filesystems = paginate(client, 'describe_file_systems', 'FileSystems')
                  backups = paginate(client, 'describe_backups', 'Backups')
              except ClientError as e:
                  code = e.response.get('Error', {}).get('Code', 'Unknown')
                  message = e.response.get('Error', {}).get('Message', str(e))
                  result['failures'].append({'operation': 'DescribeFSxResources', 'error_code': code, 'error_message': message})
                  if code in ('AccessDenied', 'AccessDeniedException', 'UnauthorizedOperation') or 'explicit deny' in message.lower():
                      result['access_denied'] += 1
                  else:
                      result['failed'] += 1
                  if not settings['continue_on_access_denied']:
                      raise
                  return result

              result['file_systems'] = len(filesystems)
              result['backups'] = len(backups)
              result['file_system_inventory'] = [fs_inventory(fs) for fs in filesystems]

              for backup in backups:
                  if settings['max_backups_per_run'] and (result['would_delete_backups'] + result['deleted_backups']) >= settings['max_backups_per_run']:
                      result['stopped_due_to_max_backups_per_run'] = True
                      return result

                  backup_id = backup.get('BackupId')
                  created = backup.get('CreationTime')
                  b_age = age_days(created)
                  arn = backup.get('ResourceARN')
                  tags, tag_error = safe_tags(client, arn)

                  item = {
                      'backup_id': backup_id,
                      'region': region,
                      'backup_type': backup.get('Type'),
                      'lifecycle': backup.get('Lifecycle'),
                      'file_system_id': backup.get('FileSystem', {}).get('FileSystemId'),
                      'file_system_type': backup.get('FileSystem', {}).get('FileSystemType'),
                      'storage_capacity_gib': backup.get('FileSystem', {}).get('StorageCapacity'),
                      'creation_time': str(created) if created else None,
                      'age_days': b_age,
                      'resource_arn': arn,
                      'tags': tags
                  }

                  if tag_error:
                      code = tag_error.response.get('Error', {}).get('Code', 'Unknown')
                      message = tag_error.response.get('Error', {}).get('Message', str(tag_error))
                      item['tag_read_error'] = {'error_code': code, 'error_message': message}
                      if code in ('AccessDenied', 'AccessDeniedException') or 'explicit deny' in message.lower():
                          result['tag_read_access_denied'] += 1
                      else:
                          result['failed'] += 1

                  if appears_aws_backup_managed(backup, tags):
                      result['aws_backup_managed_backups'] += 1

                  if not id_included(backup_id, settings['include_backup_prefixes']):
                      result['skipped_include_prefix'] += 1
                      continue

                  if id_excluded(backup_id, settings['exclude_backup_prefixes']):
                      result['skipped_exclude_prefix'] += 1
                      item['action'] = 'skipped_exclude_prefix'
                      result['candidates'].append(item)
                      continue

                  if b_age is None or b_age < settings['minimum_backup_age_days']:
                      result['skipped_too_new'] += 1
                      item['action'] = 'skipped_too_new'
                      result['candidates'].append(item)
                      continue

                  result['old_backups'] += 1

                  if settings['skip_aws_backup_managed'] and appears_aws_backup_managed(backup, tags):
                      result['skipped_aws_backup_managed'] += 1
                      item['action'] = 'skipped_aws_backup_managed'
                      result['candidates'].append(item)
                      continue

                  ignore, reason = has_ignore_tag(tags, settings['ignore_tag_keys'], settings['ignore_tag_pairs'])
                  if ignore:
                      result['skipped_ignore_tag'] += 1
                      item['action'] = 'skipped_ignore_tag'
                      item['reason'] = reason
                      result['candidates'].append(item)
                      continue

                  if not has_approval_tag(tags, settings):
                      result['skipped_missing_approval_tag'] += 1
                      item['action'] = 'skipped_missing_approval_tag'
                      item['required_tag'] = settings['approval_tag_key'] + '=' + settings['approval_tag_value']
                      result['candidates'].append(item)
                      continue

                  result['eligible'] += 1

                  if settings['action_mode'] != 'delete-approved-old-backups':
                      item['action'] = 'review_only_candidate'
                      result['candidates'].append(item)
                      continue

                  if settings['dry_run']:
                      result['would_delete_backups'] += 1
                      item['action'] = 'would_delete_backup'
                      result['candidates'].append(item)
                      continue

                  try:
                      client.delete_backup(BackupId=backup_id)
                      result['deleted_backups'] += 1
                      item['action'] = 'delete_backup'
                      result['deleted_backup_ids'].append(backup_id)
                  except ClientError as e:
                      code = e.response.get('Error', {}).get('Code', 'Unknown')
                      message = e.response.get('Error', {}).get('Message', str(e))
                      result['failures'].append({'backup_id': backup_id, 'operation': 'DeleteBackup', 'error_code': code, 'error_message': message})
                      if code in ('AccessDenied', 'AccessDeniedException') or 'explicit deny' in message.lower():
                          result['access_denied'] += 1
                      else:
                          result['failed'] += 1
                      if not settings['continue_on_access_denied']:
                          raise

              return result

          def handler(event, context):
              settings = {
                  'excluded_regions': split_csv(os.environ.get('EXCLUDED_REGIONS_CSV', '')),
                  'dry_run': env_bool('DRY_RUN', True),
                  'action_mode': os.environ.get('ACTION_MODE', 'review-only'),
                  'minimum_backup_age_days': env_int('MINIMUM_BACKUP_AGE_DAYS', 180),
                  'require_approval_tag': env_bool('REQUIRE_APPROVAL_TAG', True),
                  'approval_tag_key': os.environ.get('APPROVAL_TAG_KEY', 'CostOptApproved'),
                  'approval_tag_value': os.environ.get('APPROVAL_TAG_VALUE', 'true'),
                  'ignore_tag_keys': split_csv(os.environ.get('IGNORE_TAG_KEYS_CSV', '')),
                  'ignore_tag_pairs': parse_tag_pairs(os.environ.get('IGNORE_TAG_PAIRS_CSV', '')),
                  'include_backup_prefixes': split_csv(os.environ.get('INCLUDE_BACKUP_ID_PREFIXES_CSV', '')),
                  'exclude_backup_prefixes': split_csv(os.environ.get('EXCLUDE_BACKUP_ID_PREFIXES_CSV', '')),
                  'skip_aws_backup_managed': env_bool('SKIP_AWS_BACKUP_MANAGED_BACKUPS', True),
                  'max_backups_per_run': env_int('MAX_BACKUPS_PER_RUN', 50),
                  'continue_on_access_denied': env_bool('CONTINUE_ON_ACCESS_DENIED', True)
              }

              if isinstance(event, dict):
                  if 'dry_run' in event:
                      settings['dry_run'] = bool(event['dry_run'])
                  if 'action_mode' in event:
                      settings['action_mode'] = str(event['action_mode'])
                  if 'require_approval_tag' in event:
                      settings['require_approval_tag'] = bool(event['require_approval_tag'])
                  if 'minimum_backup_age_days' in event:
                      settings['minimum_backup_age_days'] = int(event['minimum_backup_age_days'])
                  if 'max_backups_per_run' in event:
                      settings['max_backups_per_run'] = int(event['max_backups_per_run'])
                  if 'target_regions' in event and event['target_regions']:
                      os.environ['TARGET_REGIONS_CSV'] = ','.join(event['target_regions'])

              if settings['action_mode'] not in ('review-only', 'delete-approved-old-backups'):
                  settings['action_mode'] = 'review-only'

              summary = {
                  'started_at': now().isoformat(),
                  'dry_run': settings['dry_run'],
                  'action_mode': settings['action_mode'],
                  'minimum_backup_age_days': settings['minimum_backup_age_days'],
                  'require_approval_tag': settings['require_approval_tag'],
                  'approval_tag': settings['approval_tag_key'] + '=' + settings['approval_tag_value'],
                  'skip_aws_backup_managed': settings['skip_aws_backup_managed'],
                  'max_backups_per_run': settings['max_backups_per_run'],
                  'excluded_regions': settings['excluded_regions'],
                  'regions': [],
                  'totals': {
                      'file_systems': 0,
                      'backups': 0,
                      'old_backups': 0,
                      'aws_backup_managed_backups': 0,
                      'eligible': 0,
                      'would_delete_backups': 0,
                      'deleted_backups': 0,
                      'skipped_too_new': 0,
                      'skipped_aws_backup_managed': 0,
                      'skipped_missing_approval_tag': 0,
                      'skipped_ignore_tag': 0,
                      'tag_read_access_denied': 0,
                      'access_denied': 0,
                      'failed': 0
                  },
                  'region_results': []
              }

              try:
                  regions = enabled_regions(settings['excluded_regions'])
                  summary['regions'] = regions
                  for region in regions:
                      try:
                          rr = scan_region(region, settings)
                      except Exception as e:
                          rr = {
                              'region': region,
                              'file_systems': 0,
                              'backups': 0,
                              'old_backups': 0,
                              'aws_backup_managed_backups': 0,
                              'eligible': 0,
                              'would_delete_backups': 0,
                              'deleted_backups': 0,
                              'skipped_too_new': 0,
                              'skipped_aws_backup_managed': 0,
                              'skipped_missing_approval_tag': 0,
                              'skipped_ignore_tag': 0,
                              'tag_read_access_denied': 0,
                              'access_denied': 0,
                              'failed': 1,
                              'failures': [{'error_code': type(e).__name__, 'error_message': str(e), 'trace': traceback.format_exc()}],
                              'file_system_inventory': [],
                              'candidates': [],
                              'deleted_backup_ids': []
                          }
                      summary['region_results'].append(rr)
                      for key in summary['totals']:
                          summary['totals'][key] += int(rr.get(key, 0) or 0)
              finally:
                  summary['ended_at'] = now().isoformat()
                  print(json.dumps(summary, indent=2, default=str))
              return summary

  RemediatorScheduleRule:
    Type: AWS::Events::Rule
    Condition: IsScheduleEnabled
    Properties:
      Name: !Sub '${ProjectName}-schedule'
      Description: Scheduled run for FSx File System and Backup Review Remediator.
      ScheduleExpression: !Ref ScheduleExpression
      State: ENABLED
      Targets:
        - Arn: !GetAtt RemediatorFunction.Arn
          Id: FsxBackupReviewTarget
          Input: '{"source":"eventbridge-schedule","dry_run":true,"action_mode":"review-only"}'

  RemediatorSchedulePermission:
    Type: AWS::Lambda::Permission
    Condition: IsScheduleEnabled
    Properties:
      FunctionName: !Ref RemediatorFunction
      Action: lambda:InvokeFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt RemediatorScheduleRule.Arn

Outputs:
  RemediatorFunctionName:
    Description: Lambda function name.
    Value: !Ref RemediatorFunction

  RemediatorFunctionArn:
    Description: Lambda function ARN.
    Value: !GetAtt RemediatorFunction.Arn

  DryRunSetting:
    Description: Current DryRun setting.
    Value: !Ref DryRun

  ActionModeSetting:
    Description: Current ActionMode setting.
    Value: !Ref ActionMode
