AWS Bedrock - ThrottlingException: too many tokens per day, please wait before trying again

# fallback_client.py - multi-model fallback with automatic switching
import boto3
import json
from botocore.exceptions import ClientError
import time

class BedrockFallbackClient:
    """Bedrock client with automatic fallback to alternative models."""

    def __init__(self, region="eu-west-1"):
        self.client = boto3.client("bedrock-runtime", region_name=region)
        # Fallback order: primary → secondary → tertiary
        self.models = [
            "anthropic.claude-3-5-sonnet-20241022-v2:0",  # primary
            "anthropic.claude-3-5-haiku-20241022-v1:0",   # cheaper, faster
            "anthropic.claude-3-haiku-20240307-v1:0",     # older, separate limit
            "amazon.nova-pro-v1:0",                        # Amazon model, separate limit
        ]
        self.throttled_models = {}  # model_id → timestamp when throttled

    def invoke(self, messages: list, system: str = "", max_tokens: int = 1024):
        """Invoke model with automatic fallback on throttling."""
        for model_id in self.models:
            # Skip models throttled in the last 5 minutes
            if model_id in self.throttled_models:
                if time.time() - self.throttled_models[model_id] < 300:
                    continue
                else:
                    del self.throttled_models[model_id]

            try:
                response = self.client.converse(
                    modelId=model_id,
                    messages=messages,
                    system=[{"text": system}] if system else [],
                    inferenceConfig={"maxTokens": max_tokens}
                )
                return {
                    "content": response["output"]["message"]["content"][0]["text"],
                    "model_used": model_id,
                    "input_tokens": response["usage"]["inputTokens"],
                    "output_tokens": response["usage"]["outputTokens"],
                }
            except ClientError as e:
                error_code = e.response["Error"]["Code"]
                error_msg = e.response["Error"]["Message"]

                if error_code == "ThrottlingException":
                    if "tokens per day" in error_msg.lower():
                        # Daily limit - mark model as unavailable
                        self.throttled_models[model_id] = time.time()
                        print(f"⚠️  Model {model_id} hit daily limit. Falling back...")
                        continue
                    elif "too many requests" in error_msg.lower():
                        # Per-minute limit - retry with backoff
                        time.sleep(2)
                        continue
                raise  # Other error - propagate

        raise RuntimeError("All models have reached their daily limit. No fallback available.")


# Usage:
client = BedrockFallbackClient()
result = client.invoke(
    messages=[{"role": "user", "content": [{"text": "Explain RAG architecture."}]}],
    system="You are a cloud architecture expert."
)
print(f"Model: {result['model_used']}, Tokens: {result['input_tokens']}+{result['output_tokens']}")

# token_budget_tracker.py - tracking daily budget in your application
import boto3
from datetime import datetime, timezone
import os

class TokenBudgetTracker:
    """Tracks token consumption and warns before reaching the limit."""

    def __init__(self, daily_limit: int = 2_000_000):
        self.daily_limit = daily_limit
        self.cloudwatch = boto3.client("cloudwatch")
        self._cache_timestamp = None
        self._cached_usage = 0

    def get_daily_usage(self, model_id: str) -> int:
        """Fetch current daily token usage from CloudWatch."""
        now = datetime.now(timezone.utc)
        start_of_day = now.replace(hour=0, minute=0, second=0, microsecond=0)

        # Cache for 5 minutes
        if self._cache_timestamp and (now - self._cache_timestamp).seconds < 300:
            return self._cached_usage

        input_resp = self.cloudwatch.get_metric_statistics(
            Namespace="AWS/Bedrock",
            MetricName="InputTokenCount",
            Dimensions=[{"Name": "ModelId", "Value": model_id}],
            StartTime=start_of_day,
            EndTime=now,
            Period=86400,
            Statistics=["Sum"],
        )
        output_resp = self.cloudwatch.get_metric_statistics(
            Namespace="AWS/Bedrock",
            MetricName="OutputTokenCount",
            Dimensions=[{"Name": "ModelId", "Value": model_id}],
            StartTime=start_of_day,
            EndTime=now,
            Period=86400,
            Statistics=["Sum"],
        )

        input_tokens = input_resp["Datapoints"][0]["Sum"] if input_resp["Datapoints"] else 0
        output_tokens = output_resp["Datapoints"][0]["Sum"] if output_resp["Datapoints"] else 0

        self._cached_usage = int(input_tokens + output_tokens)
        self._cache_timestamp = now
        return self._cached_usage

    def remaining_budget(self, model_id: str) -> dict:
        """Check how many tokens remain before hitting the limit."""
        used = self.get_daily_usage(model_id)
        remaining = max(0, self.daily_limit - used)
        pct_used = (used / self.daily_limit) * 100
        return {
            "used": used,
            "remaining": remaining,
            "limit": self.daily_limit,
            "pct_used": round(pct_used, 1),
            "should_fallback": pct_used > 80,
        }

# Reducing input tokens - key techniques

# 1. Shorten system prompt (biggest impact)
# BEFORE: 4000 token system prompt × 1000 queries = 4M tokens/day
# AFTER: 800 tokens (concise) × 1000 queries = 800K tokens/day

# 2. Prompt caching (Bedrock supports it for Claude 3.5+)
# System prompt sent once, cached for 5 minutes
response = client.converse(
    modelId="anthropic.claude-3-5-sonnet-20241022-v2:0",
    messages=messages,
    system=[{
        "text": long_system_prompt,
        "cacheControl": {"type": "ephemeral"}  # Cache for 5 min
    }],
)
# Cache hit = 90% reduction in cost and token usage for system prompt

# 3. Limit max_tokens in response
# Do not set max_tokens=4096 if you need a 200-token response
response = client.converse(
    modelId="anthropic.claude-3-5-sonnet-20241022-v2:0",
    messages=messages,
    inferenceConfig={"maxTokens": 512}  # Not 4096
)

# 4. Filter RAG context - send only top 3 chunks, not top 10
# BEFORE: 10 chunks × 500 tokens = 5000 tokens of context
# AFTER: 3 chunks × 500 tokens = 1500 tokens of context

# Bedrock limits are per-region. Use a second region as overflow.
import boto3
import random

class MultiRegionBedrockClient:
    """Distributes traffic across regions to double the effective limit."""

    def __init__(self, model_id: str):
        self.model_id = model_id
        self.regions = ["eu-west-1", "us-east-1"]  # Watch out for data residency!
        self.clients = {
            region: boto3.client("bedrock-runtime", region_name=region)
            for region in self.regions
        }
        self.region_index = 0

    def invoke(self, messages, system="", max_tokens=1024):
        """Round-robin between regions."""
        for attempt in range(len(self.regions)):
            region = self.regions[self.region_index % len(self.regions)]
            self.region_index += 1
            try:
                response = self.clients[region].converse(
                    modelId=self.model_id,
                    messages=messages,
                    system=[{"text": system}] if system else [],
                    inferenceConfig={"maxTokens": max_tokens}
                )
                return response
            except self.clients[region].exceptions.ThrottlingException as e:
                if "tokens per day" in str(e):
                    continue  # Try next region
                raise
        raise RuntimeError("Daily limit reached in all regions")