{
  "id": "66456674ff71",
  "url": "https://intoli.com/blog/analyzing-one-million-robots-txt-files/",
  "title": "Analyzing One Million robots.txt Files",
  "note": "This is a delightfully fun read analyzing the `robots.txt` files of the top million websites. Interesting history about the origins of the robots specification and the fact that it was never standardized. There isn't even an RFC for it! As a bonus the code excerpts to show the analysis are all in Python and was the first time I've seen the [collections](https://docs.python.org/2/library/collections.html#module-collections) module.",
  "voice": "jamie",
  "saved": "2017-09-20T09:47:53Z",
  "host": "intoli.com",
  "kind": "page",
  "tags": [],
  "record": "https://links.thingelstad.xyz/links/66456674ff71/",
  "markdown": "https://links.thingelstad.xyz/links/66456674ff71.md",
  "issues": [
    {
      "number": 20,
      "slug": "weekly-thing-for-september-23-2017",
      "title": "Weekly Thing for September 23, 2017",
      "date": "2017-09-23T13:00:00Z",
      "section": "Links 📌",
      "position": 17,
      "url": "https://weekly.thingelstad.com/archive/20/",
      "page": "https://links.thingelstad.xyz/issues/20/"
    }
  ],
  "connections": {
    "same_host": {
      "count": 0,
      "page": "https://links.thingelstad.xyz/hosts/intoli.com/",
      "nearest": []
    },
    "same_week": {
      "count": 31,
      "nearest": [
        {
          "id": "2aa8c47b1677",
          "title": "Swift.org - Swift 4.0 Released!",
          "url": "https://swift.org/blog/swift-4-0-released/",
          "record": "https://links.thingelstad.xyz/links/2aa8c47b1677/"
        },
        {
          "id": "afd8efa0c6dd",
          "title": "Satya Nadella Rewrites Microsoft’s Code",
          "url": "https://www.fastcompany.com/40457458/satya-nadella-rewrites-microsofts-code",
          "record": "https://links.thingelstad.xyz/links/afd8efa0c6dd/"
        },
        {
          "id": "14679c3e11e6",
          "title": "How Useful is Tufte for Making Maps? | Making Maps: DIY Cartography",
          "url": "https://makingmaps.net/2007/08/16/how-useful-is-tufte-for-making-maps/",
          "record": "https://links.thingelstad.xyz/links/14679c3e11e6/"
        },
        {
          "id": "bb12d68df20b",
          "title": "Facebook’s war on free will | Technology | The Guardian",
          "url": "https://www.theguardian.com/technology/2017/sep/19/facebooks-war-on-free-will",
          "record": "https://links.thingelstad.xyz/links/bb12d68df20b/"
        },
        {
          "id": "7974a268c379",
          "title": "★ Apple Watch Series 3 —Daring Fireball",
          "url": "https://daringfireball.net/2017/09/apple_watch_series_3",
          "record": "https://links.thingelstad.xyz/links/7974a268c379/"
        }
      ]
    },
    "mentions": [],
    "mentioned_by": [],
    "saved_twice": []
  },
  "elsewhere": {
    "wayback": "https://web.archive.org/web/*/https://intoli.com/blog/analyzing-one-million-robots-txt-files/"
  }
}