# Install required SDKs
pip install boto3==1.34.44 azure-storage-blob==12.19.0 google-cloud-storage==2.14.0
pip install azure-monitor-query==1.2.0 google-cloud-monitoring==2.16.0

import boto3
from datetime import datetime, timedelta
from typing import Dict, List

class S3StorageOptimizer:
    """Automated S3 storage optimization with Intelligent-Tiering and lifecycle policies."""

    def __init__(self, bucket_name: str, region: str = 'us-east-1'):
        self.s3_client = boto3.client('s3', region_name=region)
        self.cloudwatch = boto3.client('cloudwatch', region_name=region)
        self.bucket_name = bucket_name

    def enable_intelligent_tiering(self, prefix: str = "", tags: Dict[str, str] = None):
        """
        Enable S3 Intelligent-Tiering for automatic cost optimization.

        Intelligent-Tiering automatically moves objects between:
        - Frequent Access tier (< 30 days since last access)
        - Infrequent Access tier (30-90 days)
        - Archive Instant Access tier (90-180 days)
        - Archive Access tier (180-270 days, optional)
        - Deep Archive Access tier (270+ days, optional)
        """
        config_id = f"intelligent-tiering-{prefix.replace('/', '-') if prefix else 'all'}"

        # Build filter (prefix and/or tags)
        filter_config = {}
        if prefix:
            filter_config['Prefix'] = prefix
        if tags:
            filter_config['Tag'] = [{'Key': k, 'Value': v} for k, v in tags.items()]

        intelligent_tiering_config = {
            'Id': config_id,
            'Status': 'Enabled',
            'Tierings': [
                {
                    'Days': 90,
                    'AccessTier': 'ARCHIVE_ACCESS'  # Move to Archive after 90 days
                },
                {
                    'Days': 180,
                    'AccessTier': 'DEEP_ARCHIVE_ACCESS'  # Deep Archive after 180 days
                }
            ]
        }

        if filter_config:
            intelligent_tiering_config['Filter'] = filter_config

        try:
            self.s3_client.put_bucket_intelligent_tiering_configuration(
                Bucket=self.bucket_name,
                Id=config_id,
                IntelligentTieringConfiguration=intelligent_tiering_config
            )
            print(f"✓ Enabled Intelligent-Tiering: {config_id}")
            return config_id
        except Exception as e:
            print(f"✗ Failed to enable Intelligent-Tiering: {e}")
            raise

    def create_lifecycle_policy(self, policy_name: str, prefix: str = ""):
        """
        Create S3 lifecycle policy for automated data management.

        Policy actions:
        1. Move to Standard-IA after 30 days (infrequent access)
        2. Move to Glacier after 90 days (archive)
        3. Delete noncurrent versions after 30 days (versioned buckets)
        4. Delete incomplete multipart uploads after 7 days (cleanup)
        """
        lifecycle_rules = [
            {
                'ID': f'{policy_name}-transition-ia',
                'Status': 'Enabled',
                'Prefix': prefix,
                'Transitions': [
                    {
                        'Days': 30,
                        'StorageClass': 'STANDARD_IA'  # Infrequent Access after 30 days
                    },
                    {
                        'Days': 90,
                        'StorageClass': 'GLACIER_FLEXIBLE_RETRIEVAL'  # Glacier after 90 days
                    },
                    {
                        'Days': 365,
                        'StorageClass': 'DEEP_ARCHIVE'  # Deep Archive after 1 year
                    }
                ]
            },
            {
                'ID': f'{policy_name}-delete-old-versions',
                'Status': 'Enabled',
                'Prefix': prefix,
                'NoncurrentVersionTransitions': [
                    {
                        'NoncurrentDays': 30,
                        'StorageClass': 'GLACIER_FLEXIBLE_RETRIEVAL'
                    }
                ],
                'NoncurrentVersionExpiration': {
                    'NoncurrentDays': 90  # Delete old versions after 90 days
                }
            },
            {
                'ID': f'{policy_name}-cleanup-incomplete-uploads',
                'Status': 'Enabled',
                'Prefix': prefix,
                'AbortIncompleteMultipartUpload': {
                    'DaysAfterInitiation': 7  # Clean up failed uploads
                }
            }
        ]

        try:
            self.s3_client.put_bucket_lifecycle_configuration(
                Bucket=self.bucket_name,
                LifecycleConfiguration={'Rules': lifecycle_rules}
            )
            print(f"✓ Created lifecycle policy: {policy_name}")
        except Exception as e:
            print(f"✗ Failed to create lifecycle policy: {e}")
            raise

    def get_storage_metrics(self, days: int = 30) -> Dict:
        """
        Retrieve CloudWatch metrics for S3 bucket storage.

        Metrics:
        - BucketSizeBytes: Total storage used
        - NumberOfObjects: Total object count
        - Breakdown by storage class
        """
        end_time = datetime.utcnow()
        start_time = end_time - timedelta(days=days)

        metrics = {}

        # Get total bucket size
        response = self.cloudwatch.get_metric_statistics(
            Namespace='AWS/S3',
            MetricName='BucketSizeBytes',
            Dimensions=[
                {'Name': 'BucketName', 'Value': self.bucket_name},
                {'Name': 'StorageType', 'Value': 'StandardStorage'}
            ],
            StartTime=start_time,
            EndTime=end_time,
            Period=86400,  # Daily
            Statistics=['Average']
        )

        if response['Datapoints']:
            latest = max(response['Datapoints'], key=lambda x: x['Timestamp'])
            metrics['total_size_gb'] = latest['Average'] / (1024**3)

        # Get object count
        response = self.cloudwatch.get_metric_statistics(
            Namespace='AWS/S3',
            MetricName='NumberOfObjects',
            Dimensions=[
                {'Name': 'BucketName', 'Value': self.bucket_name},
                {'Name': 'StorageType', 'Value': 'AllStorageTypes'}
            ],
            StartTime=start_time,
            EndTime=end_time,
            Period=86400,
            Statistics=['Average']
        )

        if response['Datapoints']:
            latest = max(response['Datapoints'], key=lambda x: x['Timestamp'])
            metrics['object_count'] = int(latest['Average'])

        return metrics

    def analyze_storage_costs(self) -> Dict:
        """
        Analyze current storage distribution and estimated costs.

        Uses S3 Storage Lens or inventory reports for detailed analysis.
        """
        # Get storage class distribution using S3 Inventory or List Objects
        storage_classes = {}
        paginator = self.s3_client.get_paginator('list_objects_v2')

        for page in paginator.paginate(Bucket=self.bucket_name):
            if 'Contents' not in page:
                continue

            for obj in page['Contents']:
                storage_class = obj.get('StorageClass', 'STANDARD')
                size_bytes = obj['Size']

                if storage_class not in storage_classes:
                    storage_classes[storage_class] = {'count': 0, 'size_bytes': 0}

                storage_classes[storage_class]['count'] += 1
                storage_classes[storage_class]['size_bytes'] += size_bytes

        # Calculate estimated monthly costs (approximate pricing)
        pricing = {
            'STANDARD': 0.023,
            'STANDARD_IA': 0.0125,
            'INTELLIGENT_TIERING': 0.023,
            'GLACIER': 0.0036,
            'DEEP_ARCHIVE': 0.00099
        }

        total_cost = 0
        for storage_class, data in storage_classes.items():
            size_gb = data['size_bytes'] / (1024**3)
            cost = size_gb * pricing.get(storage_class, 0.023)
            storage_classes[storage_class]['size_gb'] = size_gb
            storage_classes[storage_class]['estimated_monthly_cost'] = cost
            total_cost += cost

        return {
            'storage_classes': storage_classes,
            'total_monthly_cost_usd': total_cost
        }


# Example usage
if __name__ == "__main__":
    optimizer = S3StorageOptimizer(bucket_name='my-production-bucket')

    # Enable Intelligent-Tiering for log files
    optimizer.enable_intelligent_tiering(
        prefix="logs/",
        tags={'Environment': 'Production', 'DataType': 'Logs'}
    )

    # Create lifecycle policy for backups
    optimizer.create_lifecycle_policy(
        policy_name='backup-retention',
        prefix='backups/'
    )

    # Analyze current storage costs
    analysis = optimizer.analyze_storage_costs()
    print(f"\nStorage Cost Analysis:")
    print(f"Total Monthly Cost: ${analysis['total_monthly_cost_usd']:.2f}")

    for storage_class, data in analysis['storage_classes'].items():
        print(f"\n{storage_class}:")
        print(f"  Objects: {data['count']:,}")
        print(f"  Size: {data['size_gb']:.2f} GB")
        print(f"  Monthly Cost: ${data['estimated_monthly_cost']:.2f}")

def create_storage_alarms(bucket_name: str, threshold_gb: float = 1000):
    """Create CloudWatch alarms for S3 storage growth."""
    cloudwatch = boto3.client('cloudwatch')
    sns = boto3.client('sns')

    # Create SNS topic for alerts
    topic_response = sns.create_topic(Name=f'{bucket_name}-storage-alerts')
    topic_arn = topic_response['TopicArn']

    # Subscribe email to topic (replace with your email)
    sns.subscribe(
        TopicArn=topic_arn,
        Protocol='email',
        Endpoint='[email protected]'
    )

    # Create alarm for storage growth
    cloudwatch.put_metric_alarm(
        AlarmName=f'{bucket_name}-storage-exceeds-{threshold_gb}GB',
        ComparisonOperator='GreaterThanThreshold',
        EvaluationPeriods=1,
        MetricName='BucketSizeBytes',
        Namespace='AWS/S3',
        Period=86400,  # Daily check
        Statistic='Average',
        Threshold=threshold_gb * (1024**3),  # Convert GB to bytes
        ActionsEnabled=True,
        AlarmActions=[topic_arn],
        AlarmDescription=f'Alert when {bucket_name} exceeds {threshold_gb}GB',
        Dimensions=[
            {'Name': 'BucketName', 'Value': bucket_name},
            {'Name': 'StorageType', 'Value': 'StandardStorage'}
        ]
    )

    print(f"✓ Created storage alarm with threshold {threshold_gb}GB")
    print(f"✓ Alerts will be sent to: {topic_arn}")

from azure.storage.blob import BlobServiceClient, BlobClient
from azure.mgmt.storage import StorageManagementClient
from azure.mgmt.storage.models import ManagementPolicy, ManagementPolicyRule, ManagementPolicyDefinition
from azure.identity import DefaultAzureCredential
from datetime import datetime, timedelta

class AzureBlobStorageOptimizer:
    """Automated Azure Blob Storage optimization with lifecycle management."""

    def __init__(self, account_name: str, resource_group: str, subscription_id: str):
        self.account_name = account_name
        self.resource_group = resource_group
        self.subscription_id = subscription_id

        # Use DefaultAzureCredential for authentication (supports managed identity, Azure CLI, etc.)
        credential = DefaultAzureCredential()

        self.blob_service_client = BlobServiceClient(
            account_url=f"https://{account_name}.blob.core.windows.net",
            credential=credential
        )

        self.storage_mgmt_client = StorageManagementClient(
            credential=credential,
            subscription_id=subscription_id
        )

    def create_lifecycle_policy(self):
        """
        Create Azure Blob Storage lifecycle management policy.

        Tier transitions:
        - Hot (default): Frequently accessed
        - Cool (30+ days): Infrequent access ($0.01/GB)
        - Archive (90+ days): Rarely accessed ($0.00099/GB, retrieval time: hours)
        """
        lifecycle_policy = ManagementPolicy(
            policy=ManagementPolicyDefinition(
                rules=[
                    ManagementPolicyRule(
                        name="move-to-cool-tier",
                        enabled=True,
                        type="Lifecycle",
                        definition={
                            "filters": {
                                "blobTypes": ["blockBlob"],
                                "prefixMatch": ["logs/", "backups/"]
                            },
                            "actions": {
                                "baseBlob": {
                                    "tierToCool": {
                                        "daysAfterModificationGreaterThan": 30
                                    },
                                    "tierToArchive": {
                                        "daysAfterModificationGreaterThan": 90
                                    },
                                    "delete": {
                                        "daysAfterModificationGreaterThan": 365
                                    }
                                },
                                "snapshot": {
                                    "delete": {
                                        "daysAfterCreationGreaterThan": 90
                                    }
                                }
                            }
                        }
                    ),
                    ManagementPolicyRule(
                        name="delete-old-logs",
                        enabled=True,
                        type="Lifecycle",
                        definition={
                            "filters": {
                                "blobTypes": ["blockBlob"],
                                "prefixMatch": ["application-logs/"]
                            },
                            "actions": {
                                "baseBlob": {
                                    "delete": {
                                        "daysAfterModificationGreaterThan": 90
                                    }
                                }
                            }
                        }
                    )
                ]
            )
        )

        # Apply policy to storage account
        self.storage_mgmt_client.management_policies.create_or_update(
            resource_group_name=self.resource_group,
            account_name=self.account_name,
            management_policy_name='default',
            properties=lifecycle_policy
        )

        print("✓ Created Azure Blob Storage lifecycle policy")

    def analyze_storage_tiers(self, container_name: str) -> Dict:
        """Analyze blob distribution across access tiers."""
        container_client = self.blob_service_client.get_container_client(container_name)

        tier_stats = {
            'Hot': {'count': 0, 'size_bytes': 0},
            'Cool': {'count': 0, 'size_bytes': 0},
            'Archive': {'count': 0, 'size_bytes': 0}
        }

        blob_list = container_client.list_blobs(include=['metadata'])

        for blob in blob_list:
            tier = blob.blob_tier or 'Hot'
            tier_stats[tier]['count'] += 1
            tier_stats[tier]['size_bytes'] += blob.size

        # Calculate costs (approximate Azure pricing)
        pricing = {
            'Hot': 0.018,    # per GB/month
            'Cool': 0.01,
            'Archive': 0.00099
        }

        for tier, data in tier_stats.items():
            size_gb = data['size_bytes'] / (1024**3)
            data['size_gb'] = size_gb
            data['monthly_cost_usd'] = size_gb * pricing[tier]

        return tier_stats

    def set_blob_tier(self, container_name: str, blob_name: str, tier: str):
        """
        Manually set blob access tier for immediate optimization.

        Tiers: 'Hot', 'Cool', 'Archive'
        """
        blob_client = self.blob_service_client.get_blob_client(
            container=container_name,
            blob=blob_name
        )

        blob_client.set_standard_blob_tier(tier)
        print(f"✓ Set {blob_name} to {tier} tier")


# Example usage
if __name__ == "__main__":
    optimizer = AzureBlobStorageOptimizer(
        account_name='mystorageaccount',
        resource_group='my-resource-group',
        subscription_id='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
    )

    # Create lifecycle management policy
    optimizer.create_lifecycle_policy()

    # Analyze storage tier distribution
    tier_analysis = optimizer.analyze_storage_tiers(container_name='production-data')

    print("\nStorage Tier Analysis:")
    for tier, data in tier_analysis.items():
        print(f"{tier}: {data['count']} blobs, {data['size_gb']:.2f} GB, ${data['monthly_cost_usd']:.2f}/month")

from google.cloud import storage
from google.cloud.storage import Bucket
from datetime import datetime, timedelta

class GCSStorageOptimizer:
    """Automated Google Cloud Storage optimization with lifecycle policies."""

    def __init__(self, bucket_name: str, project_id: str):
        self.client = storage.Client(project=project_id)
        self.bucket = self.client.bucket(bucket_name)

    def create_lifecycle_policy(self):
        """
        Create GCS lifecycle management policy.

        Storage classes:
        - Standard: Frequently accessed ($0.020/GB)
        - Nearline: < 1/month access ($0.010/GB)
        - Coldline: < 1/quarter access ($0.004/GB)
        - Archive: < 1/year access ($0.0012/GB)
        """
        lifecycle_rules = [
            {
                "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"},
                "condition": {
                    "age": 30,  # Days since upload
                    "matchesPrefix": ["logs/", "backups/"]
                }
            },
            {
                "action": {"type": "SetStorageClass", "storageClass": "COLDLINE"},
                "condition": {
                    "age": 90,
                    "matchesPrefix": ["logs/", "backups/"]
                }
            },
            {
                "action": {"type": "SetStorageClass", "storageClass": "ARCHIVE"},
                "condition": {
                    "age": 365,
                    "matchesPrefix": ["archives/"]
                }
            },
            {
                "action": {"type": "Delete"},
                "condition": {
                    "age": 730,  # Delete after 2 years
                    "matchesPrefix": ["temp/"]
                }
            },
            {
                "action": {"type": "Delete"},
                "condition": {
                    "numNewerVersions": 3  # Keep only 3 latest versions (versioned buckets)
                }
            }
        ]

        self.bucket.lifecycle_rules = lifecycle_rules
        self.bucket.patch()

        print(f"✓ Created GCS lifecycle policy for bucket: {self.bucket.name}")

    def analyze_storage_classes(self) -> Dict:
        """Analyze object distribution across storage classes."""
        storage_classes = {}

        blobs = self.client.list_blobs(self.bucket.name)

        for blob in blobs:
            storage_class = blob.storage_class or 'STANDARD'

            if storage_class not in storage_classes:
                storage_classes[storage_class] = {'count': 0, 'size_bytes': 0}

            storage_classes[storage_class]['count'] += 1
            storage_classes[storage_class]['size_bytes'] += blob.size

        # Calculate costs (GCS pricing)
        pricing = {
            'STANDARD': 0.020,
            'NEARLINE': 0.010,
            'COLDLINE': 0.004,
            'ARCHIVE': 0.0012
        }

        for storage_class, data in storage_classes.items():
            size_gb = data['size_bytes'] / (1024**3)
            data['size_gb'] = size_gb
            data['monthly_cost_usd'] = size_gb * pricing.get(storage_class, 0.020)

        return storage_classes

    def enable_autoclass(self):
        """
        Enable GCS Autoclass for automatic storage class optimization.

        Autoclass automatically transitions objects to optimal storage classes
        based on access patterns (similar to S3 Intelligent-Tiering).
        """
        self.bucket.autoclass_enabled = True
        self.bucket.autoclass_terminal_storage_class = "ARCHIVE"
        self.bucket.patch()

        print(f"✓ Enabled Autoclass for bucket: {self.bucket.name}")


# Example usage
if __name__ == "__main__":
    optimizer = GCSStorageOptimizer(
        bucket_name='my-production-bucket',
        project_id='my-gcp-project'
    )

    # Enable Autoclass (recommended for unknown access patterns)
    optimizer.enable_autoclass()

    # Or create custom lifecycle policy
    # optimizer.create_lifecycle_policy()

    # Analyze storage class distribution
    analysis = optimizer.analyze_storage_classes()

    print("\nGCS Storage Class Analysis:")
    total_cost = 0
    for storage_class, data in analysis.items():
        print(f"{storage_class}: {data['count']} objects, {data['size_gb']:.2f} GB, ${data['monthly_cost_usd']:.2f}/month")
        total_cost += data['monthly_cost_usd']

    print(f"\nTotal Monthly Cost: ${total_cost:.2f}")

def create_comprehensive_monitoring(bucket_name: str, provider: str = 'aws'):
    """Set up monitoring dashboards and alerts for storage optimization."""

    if provider == 'aws':
        cloudwatch = boto3.client('cloudwatch')

        # Create custom dashboard
        dashboard_body = {
            "widgets": [
                {
                    "type": "metric",
                    "properties": {
                        "metrics": [
                            ["AWS/S3", "BucketSizeBytes", {"stat": "Average"}]
                        ],
                        "period": 86400,
                        "stat": "Average",
                        "region": "us-east-1",
                        "title": "S3 Storage Growth"
                    }
                }
            ]
        }

        cloudwatch.put_dashboard(
            DashboardName=f'{bucket_name}-storage-dashboard',
            DashboardBody=str(dashboard_body)
        )

        print(f"✓ Created CloudWatch dashboard: {bucket_name}-storage-dashboard")

def run_storage_audit(optimizer):
    """Run comprehensive storage audit and generate recommendations."""

    analysis = optimizer.analyze_storage_costs()

    print("\n=== Storage Audit Report ===")
    print(f"Total Monthly Cost: ${analysis['total_monthly_cost_usd']:.2f}")

    # Identify optimization opportunities
    recommendations = []

    for storage_class, data in analysis['storage_classes'].items():
        if storage_class == 'STANDARD' and data['count'] > 1000:
            potential_savings = data['size_gb'] * (0.023 - 0.0125)  # Standard to IA
            recommendations.append({
                'action': f"Move old {storage_class} objects to Standard-IA",
                'potential_monthly_savings': potential_savings
            })

    if recommendations:
        print("\nOptimization Recommendations:")
        for rec in recommendations:
            print(f"- {rec['action']}: Save ${rec['potential_monthly_savings']:.2f}/month")

    return recommendations

def tag_storage_for_cost_allocation(bucket_name: str, tags: Dict[str, str]):
    """Apply cost allocation tags to S3 bucket for granular billing."""
    s3_client = boto3.client('s3')

    tagging = {'TagSet': [{'Key': k, 'Value': v} for k, v in tags.items()]}

    s3_client.put_bucket_tagging(
        Bucket=bucket_name,
        Tagging=tagging
    )

    print(f"✓ Applied cost allocation tags to {bucket_name}")
    print(f"  Tags: {tags}")

# Example usage
tag_storage_for_cost_allocation(
    bucket_name='production-data',
    tags={
        'Environment': 'Production',
        'CostCenter': 'Engineering',
        'Application': 'DataPipeline',
        'Owner': '[email protected]'
    }
)

def enable_cross_region_replication(source_bucket: str, dest_bucket: str, dest_region: str):
    """Enable S3 Cross-Region Replication for disaster recovery and performance."""
    s3_client = boto3.client('s3')

    # Enable versioning (required for replication)
    s3_client.put_bucket_versioning(
        Bucket=source_bucket,
        VersioningConfiguration={'Status': 'Enabled'}
    )

    s3_client.put_bucket_versioning(
        Bucket=dest_bucket,
        VersioningConfiguration={'Status': 'Enabled'}
    )

    # Create IAM role for replication (simplified)
    replication_config = {
        'Role': 'arn:aws:iam::ACCOUNT_ID:role/s3-replication-role',
        'Rules': [
            {
                'ID': 'replicate-all',
                'Priority': 1,
                'Status': 'Enabled',
                'Filter': {'Prefix': ''},
                'Destination': {
                    'Bucket': f'arn:aws:s3:::{dest_bucket}',
                    'ReplicationTime': {
                        'Status': 'Enabled',
                        'Time': {'Minutes': 15}
                    },
                    'Metrics': {
                        'Status': 'Enabled',
                        'EventThreshold': {'Minutes': 15}
                    }
                },
                'DeleteMarkerReplication': {'Status': 'Enabled'}
            }
        ]
    }

    s3_client.put_bucket_replication(
        Bucket=source_bucket,
        ReplicationConfiguration=replication_config
    )

    print(f"✓ Enabled replication: {source_bucket} -> {dest_bucket} ({dest_region})")

def diagnose_lifecycle_issues(bucket_name: str):
    """Check if lifecycle policies are configured correctly."""
    s3_client = boto3.client('s3')

    # Check lifecycle configuration
    try:
        response = s3_client.get_bucket_lifecycle_configuration(Bucket=bucket_name)
        print(f"✓ Lifecycle policies found: {len(response['Rules'])} rules")

        for rule in response['Rules']:
            print(f"\nRule: {rule['ID']}")
            print(f"  Status: {rule['Status']}")
            print(f"  Prefix: {rule.get('Filter', {}).get('Prefix', 'All objects')}")
    except s3_client.exceptions.NoSuchLifecycleConfiguration:
        print("✗ No lifecycle policies configured!")
        return False

    # Check Intelligent-Tiering configuration
    try:
        response = s3_client.list_bucket_intelligent_tiering_configurations(Bucket=bucket_name)
        if response.get('IntelligentTieringConfigurationList'):
            print(f"\n✓ Intelligent-Tiering enabled: {len(response['IntelligentTieringConfigurationList'])} configs")
    except Exception as e:
        print(f"✗ Intelligent-Tiering not configured: {e}")

    return True

def check_archive_retrieval_status(bucket_name: str, object_key: str):
    """Check if archived object is being restored."""
    s3_client = boto3.client('s3')

    response = s3_client.head_object(Bucket=bucket_name, Key=object_key)

    if 'Restore' in response:
        print(f"Restore Status: {response['Restore']}")
        if 'ongoing-request="true"' in response['Restore']:
            print("⏳ Restoration in progress...")
        elif 'ongoing-request="false"' in response['Restore']:
            print("✓ Restoration complete!")
    else:
        print("Object not in archive or not being restored")