{
  "id": "153e1a3e02fe",
  "url": "https://manuelmoreale.com/bots-spiders-and-crawlers-the-results",
  "title": "Bots, Spiders, and Crawlers: The Results – Manu",
  "note": "I've long assume robots.txt is ignored by most robots. Moreale's experiment is good proof of that. If you really want to blog robots, you'll need to do it on the HTTP server and it is going to be a big pain.",
  "voice": "jamie",
  "saved": "2023-09-28T12:53:55Z",
  "host": "manuelmoreale.com",
  "kind": "page",
  "tags": [
    "_brief"
  ],
  "record": "https://links.thingelstad.xyz/links/153e1a3e02fe/",
  "markdown": "https://links.thingelstad.xyz/links/153e1a3e02fe.md",
  "issues": [
    {
      "number": 263,
      "slug": "263",
      "title": "Weekly Thing 263 / Copilot, Del.icio.us, Pancake",
      "date": "2023-09-30T12:00:00Z",
      "section": "Briefly",
      "position": 15,
      "url": "https://weekly.thingelstad.com/archive/263/",
      "page": "https://links.thingelstad.xyz/issues/263/"
    }
  ],
  "connections": {
    "same_host": {
      "count": 7,
      "page": "https://links.thingelstad.xyz/hosts/manuelmoreale.com/",
      "nearest": [
        {
          "id": "661ee39b7484",
          "title": "Places on the web – Manu",
          "url": "https://manuelmoreale.com/places-on-the-web",
          "record": "https://links.thingelstad.xyz/links/661ee39b7484/"
        },
        {
          "id": "2f42dc24ca48",
          "title": "On Ad Blockers – Manu",
          "url": "https://manuelmoreale.com/on-ad-blockers",
          "record": "https://links.thingelstad.xyz/links/2f42dc24ca48/"
        },
        {
          "id": "7a87e8d1a417",
          "title": "On POSSE – Manu",
          "url": "https://manuelmoreale.com/on-posse",
          "record": "https://links.thingelstad.xyz/links/7a87e8d1a417/"
        },
        {
          "id": "92c55991626e",
          "title": "The web is not dying – Manu",
          "url": "https://manuelmoreale.com/the-web-is-not-dying",
          "record": "https://links.thingelstad.xyz/links/92c55991626e/"
        },
        {
          "id": "237ba9b63f74",
          "title": "On hobbies, side projects, and money – Manu",
          "url": "https://manuelmoreale.com/on-hobbies-side-projects-and-money",
          "record": "https://links.thingelstad.xyz/links/237ba9b63f74/"
        }
      ]
    },
    "same_week": {
      "count": 16,
      "nearest": [
        {
          "id": "dcdff045f9ac",
          "title": "The Screens are the Symptom | danah boyd",
          "url": "http://www.zephoria.org/thoughts/archives/2023/09/26/the-screens-are-the-symptom.html",
          "record": "https://links.thingelstad.xyz/links/dcdff045f9ac/"
        },
        {
          "id": "a404cc0abfad",
          "title": "macOS Sonoma: The MacStories Review - MacStories",
          "url": "https://www.macstories.net/stories/macos-sonoma-the-macstories-review/",
          "record": "https://links.thingelstad.xyz/links/a404cc0abfad/"
        },
        {
          "id": "af210437ecf5",
          "title": "How Mastercard sells its ‘gold mine’ of transaction data",
          "url": "https://pirg.org/edfund/resources/how-mastercard-sells-data/",
          "record": "https://links.thingelstad.xyz/links/af210437ecf5/"
        },
        {
          "id": "ee67517f3bb6",
          "title": "The World’s Best Pancake Recipe",
          "url": "https://kottke.org/09/10/the-worlds-best-pancake-recipe",
          "record": "https://links.thingelstad.xyz/links/ee67517f3bb6/"
        },
        {
          "id": "9b485f67ec25",
          "title": "Using ChatGPT as an editor – recursion by Luke Francl",
          "url": "http://recursion.org/2023/9/27/chatgpt-editor-strange-loop",
          "record": "https://links.thingelstad.xyz/links/9b485f67ec25/"
        }
      ]
    },
    "mentions": [],
    "mentioned_by": [],
    "saved_twice": []
  },
  "elsewhere": {
    "wayback": "https://web.archive.org/web/*/https://manuelmoreale.com/bots-spiders-and-crawlers-the-results"
  }
}