Robots.txt Crawler Rules Parser

{
  "type": "object",
  "required": [
    "url"
  ],
  "properties": {
    "url": {
      "type": "string",
      "description": "The website URL to fetch robots.txt for, e.g. 'https://example.com'."
    }
  },
  "additionalProperties": false
}

{
  "type": "object",
  "properties": {
    "rules": {
      "type": "array",
      "items": {
        "type": "object",
        "required": [
          "user_agent",
          "allow",
          "disallow"
        ],
        "properties": {
          "allow": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Path patterns this user-agent is explicitly allowed. Often empty."
          },
          "disallow": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Path patterns this user-agent is disallowed. Often empty."
          },
          "user_agent": {
            "type": "string"
          },
          "crawl_delay": {
            "type": "number",
            "description": "Seconds between requests this user-agent is asked to wait. Only present on rules that declare one -- observed on 'SemrushBot' (5), 'PerplexityBot' (1), and 'baidu' (1); absent (not null) on every other rule observed."
          }
        },
        "additionalProperties": true
      },
      "description": "One entry per named user-agent, in file order."
    },
    "exists": {
      "type": "boolean",
      "description": "Whether the site served a robots.txt file at all. Observed true on both real fixtures."
    },
    "raw_length": {
      "type": "number",
      "description": "Byte length of the raw robots.txt file that was fetched and parsed."
    },
    "total_rules": {
      "type": "number",
      "description": "Total number of per-user-agent rule blocks parsed (34 and 13 observed)."
    },
    "user_agents": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Every user-agent named in the file, in file order, including the wildcard '*' entry when present."
    },
    "total_allow_rules": {
      "type": "number",
      "description": "Total count of allow-path entries across all rules."
    },
    "blocks_all_crawlers": {
      "type": "boolean",
      "description": "True when the file's rules disallow every crawler outright. Observed false on both real fixtures captured; a genuinely blocking robots.txt was not observed."
    },
    "total_disallow_rules": {
      "type": "number",
      "description": "Total count of disallow-path entries across all rules."
    }
  },
  "description": "The parsed robots.txt result, as returned at routing.responseResultPath ($.output). Both real fixtures captured have `exists: true`; the shape when a site has no robots.txt file is unconfirmed (see verification.notes).",
  "additionalProperties": true
}