{
  "id": "f8bf443a8165",
  "url": "https://aws.amazon.com/message/101925/",
  "title": "Summary of the Amazon DynamoDB Service Disruption in Northern Virginia (US-EAST-1) Region",
  "note": "On Monday, Oct 20, AWS had a massive failure that took many services across the Internet offline. The core issue was a failure in DNS automation.\n\n> Many of the largest AWS services rely extensively on DNS to provide seamless scale, fault isolation and recovery, low latency, and locality. Services like DynamoDB maintain hundreds of thousands of DNS records to operate a very large heterogeneous fleet of load balancers in each Region. Automation is crucial to ensuring that these DNS records are updated frequently to add additional capacity as it becomes available, to correctly handle hardware failures, and to efficiently distribute traffic to optimize customers' experience. […] **The root cause of this issue was a latent race condition in the DynamoDB DNS management system that resulted in an incorrect empty DNS record for the service's regional endpoint (dynamodb.us-east-1.amazonaws.com) that the automation failed to repair.**\n\nWithout DNS no system knows how to find any other systems. Additionally AWS automation for many other services relies on DynamoDB. This was a particularly bad failure state. \n",
  "voice": "mixed",
  "saved": "2025-10-24T03:21:16Z",
  "host": "aws.amazon.com",
  "kind": "page",
  "tags": [],
  "record": "https://links.thingelstad.xyz/links/f8bf443a8165/",
  "markdown": "https://links.thingelstad.xyz/links/f8bf443a8165.md",
  "issues": [
    {
      "number": 331,
      "slug": "331",
      "title": "Weekly Thing 331 / RFC, Security, Tokens",
      "date": "2025-11-02T12:00:00Z",
      "section": "Notable",
      "position": 3,
      "url": "https://weekly.thingelstad.com/archive/331/",
      "page": "https://links.thingelstad.xyz/issues/331/"
    }
  ],
  "connections": {
    "same_host": {
      "count": 107,
      "page": "https://links.thingelstad.xyz/hosts/aws.amazon.com/",
      "nearest": [
        {
          "id": "92c00f498539",
          "title": "Introducing Amazon Bedrock AgentCore: Securely deploy and operate AI agents at any scale (preview) | AWS News Blog",
          "url": "https://aws.amazon.com/blogs/aws/introducing-amazon-bedrock-agentcore-securely-deploy-and-operate-ai-agents-at-any-scale/",
          "record": "https://links.thingelstad.xyz/links/92c00f498539/"
        },
        {
          "id": "74d34753a948",
          "title": "Introducing AWS Lambda Managed Instances: Serverless simplicity with EC2 flexibility | AWS News Blog",
          "url": "https://aws.amazon.com/blogs/aws/introducing-aws-lambda-managed-instances-serverless-simplicity-with-ec2-flexibility/",
          "record": "https://links.thingelstad.xyz/links/74d34753a948/"
        },
        {
          "id": "10a2424966c4",
          "title": "Amazon S3 Vectors now generally available with increased scale and performance | AWS News Blog",
          "url": "https://aws.amazon.com/blogs/aws/amazon-s3-vectors-now-generally-available-with-increased-scale-and-performance/",
          "record": "https://links.thingelstad.xyz/links/10a2424966c4/"
        },
        {
          "id": "eb3bc6487a1f",
          "title": "Introducing OpenClaw on Amazon Lightsail to run your autonomous private AI agents | AWS News Blog",
          "url": "https://aws.amazon.com/blogs/aws/introducing-openclaw-on-amazon-lightsail-to-run-your-autonomous-private-ai-agents/",
          "record": "https://links.thingelstad.xyz/links/eb3bc6487a1f/"
        },
        {
          "id": "d79a0cb89f95",
          "title": "Four security principles for agentic AI systems | AWS Security Blog",
          "url": "https://aws.amazon.com/blogs/security/four-security-principles-for-agentic-ai-systems/",
          "record": "https://links.thingelstad.xyz/links/d79a0cb89f95/"
        }
      ]
    },
    "same_week": {
      "count": 17,
      "nearest": [
        {
          "id": "db6aabd7a5b7",
          "title": "Scripts I wrote that I use all the time",
          "url": "https://evanhahn.com/scripts-i-wrote-that-i-use-all-the-time/",
          "record": "https://links.thingelstad.xyz/links/db6aabd7a5b7/"
        },
        {
          "id": "4bb00881134e",
          "title": "Disco - Self-hosting that doesn't feel like self-hosting",
          "url": "https://disco.cloud/",
          "record": "https://links.thingelstad.xyz/links/4bb00881134e/"
        },
        {
          "id": "9b2e1ea827a5",
          "title": "Measuring Engineering Productivity - Off By One",
          "url": "https://justoffbyone.com/posts/measuring-engineering-productivity/",
          "record": "https://links.thingelstad.xyz/links/9b2e1ea827a5/"
        },
        {
          "id": "0df3519b85ac",
          "title": "Space Exploration Logo Archive",
          "url": "https://spaceexplorationlogoarchive.webflow.io/",
          "record": "https://links.thingelstad.xyz/links/0df3519b85ac/"
        },
        {
          "id": "3abc39b01fb7",
          "title": "Rivian’s Also e-bike is like nothing you’ve ever seen before | The Verge",
          "url": "https://www.theverge.com/news/804157/rivian-tm-b-electric-bike-price-specs-helmet-quad",
          "record": "https://links.thingelstad.xyz/links/3abc39b01fb7/"
        }
      ]
    },
    "mentions": [],
    "mentioned_by": [],
    "saved_twice": []
  },
  "elsewhere": {
    "wayback": "https://web.archive.org/web/*/https://aws.amazon.com/message/101925/"
  }
}