{
  "study": "AI crawlery na českých firemních webech",
  "publisher": "CIAD - Cesky institut pro AI a data",
  "url": "https://ciad.cz/studie/ai-crawlery-ceske-weby/",
  "license": "CC BY 4.0",
  "generated": "2026-08-03",
  "scan_date": "2026-08-03",
  "note_on_data": "Agregatni data bez domenovych jmen a bez jmen firem. Vzorek je konvenicni (vyzkumny korpus CIAD pouzivany napric paralelnimi bezpecnostnimi studiemi), ne nahodny nebo reprezentativni vzorek ceske ekonomiky.",
  "sample": {
    "total_domains_targeted": 299,
    "n_unreachable_excluded": 39,
    "n_reachable_robots_denominator": 260,
    "n_domains_with_parseable_robots_txt": 236,
    "n_tls_handshake_ok": 257,
    "sampling_method": "convenience sample: CIAD research corpus used across parallel security studies, not random or representative of the Czech economy"
  },
  "robots_policy_bucket": {
    "denominator": 260,
    "counts": {
      "silent_no_ai_specific_rules": 217,
      "no_robots_txt": 24,
      "explicit_block_only": 12,
      "explicit_allow_only": 3,
      "explicit_mixed": 2,
      "wildcard_block_all": 2
    },
    "pct": {
      "silent_no_ai_specific_rules": 83.5,
      "no_robots_txt": 9.2,
      "explicit_block_only": 4.6,
      "explicit_allow_only": 1.2,
      "explicit_mixed": 0.8,
      "wildcard_block_all": 0.8
    },
    "no_ai_specific_decision_combined": {
      "n": 241,
      "pct": 92.7,
      "definition": "silent_no_ai_specific_rules + no_robots_txt"
    }
  },
  "deliberate_choice_signal": {
    "definition": "domains with at least one AI crawler named explicitly in a Disallow rule, excluding generic wildcard rules that apply equally to every bot including Googlebot",
    "denominator_domains_with_robots_txt": 236,
    "n_named_ai_crawler_blockers": 15,
    "n_blocks_ai_but_allows_googlebot": 9,
    "pct_of_named_blockers_that_still_allow_googlebot": 60.0,
    "reconciliation_with_robots_policy_bucket": "14 of these 15 domains are also the primary robots_policy_bucket classification (explicit_block_only 12 + explicit_mixed 2). The 15th domain also carries a named AI-crawler Disallow rule but is primarily classified as wildcard_block_all (1 of the 2 domains in that bucket), because a blanket `User-agent: * Disallow: /` rule takes classification precedence. It is not double-counted in the 260-domain policy table, only in this separately-defined deliberate-choice metric."
  },
  "per_ai_crawler_named_block": {
    "denominator_domains_with_robots_txt": 236,
    "note": "one domain may name more than one crawler, so the column sum exceeds 15",
    "counts": {
      "Amazonbot": 14,
      "ClaudeBot": 12,
      "GPTBot": 11,
      "Google-Extended": 11,
      "Applebot-Extended": 10,
      "Bytespider": 9,
      "meta-externalagent": 9,
      "CCBot": 9,
      "PerplexityBot": 4,
      "anthropic-ai": 3,
      "OAI-SearchBot": 3,
      "ChatGPT-User": 3,
      "Claude-User": 2,
      "Claude-SearchBot": 2,
      "Perplexity-User": 2
    }
  },
  "tls": {
    "n_targeted": 299,
    "n_handshake_ok": 257,
    "outcome_breakdown": {
      "dns_failure": 34,
      "timeout": 5,
      "connection_refused": 3
    },
    "protocol_breakdown": {
      "TLSv1.3": {
        "n": 223,
        "pct": 86.8
      },
      "TLSv1.2": {
        "n": 34,
        "pct": 13.2
      }
    },
    "legacy_tls_1_0_1_1_acceptance": "NOT_TESTABLE_FROM_THIS_ENVIRONMENT - local OpenSSL 3.6.2 has TLS1.0/1.1 disabled at the library level, applies equally to every target, not a per-domain result",
    "chain_validates": {
      "n": 247,
      "pct": 96.1
    },
    "hostname_mismatch": 6,
    "already_expired": 2,
    "expiry_days_median_among_handshake_ok": 65,
    "expiry_within_30_days": 12,
    "issuer_breakdown_n_ge_10": {
      "Let's Encrypt": {
        "n": 181,
        "pct": 70.4
      },
      "Google Trust Services": {
        "n": 29,
        "pct": 11.3
      },
      "DigiCert Inc": {
        "n": 15,
        "pct": 5.8
      },
      "other_lt10_combined": {
        "n": 32,
        "pct": 12.5
      }
    }
  },
  "http_protocol": {
    "denominator_tls_handshake_ok": 257,
    "http2_alpn_supported": {
      "n": 226,
      "pct": 87.9
    },
    "denominator_reachable_with_headers": 260,
    "http3_advertised_via_alt_svc": {
      "n": 75,
      "pct": 28.8
    },
    "caveat": "ALPN reflects server capability, not confirmation the scan's own GET used h2. HTTP/3 measured only via Alt-Svc header advertisement on the /robots.txt response, no QUIC handshake performed. Denominator is 260 (reachable domains with response headers available), not 299 - an unreachable domain sends no headers and must not be counted as 'no HTTP/3'."
  },
  "version_disclosure": {
    "denominator_headers_available": 260,
    "server_header_with_version": {
      "n": 30,
      "pct": 11.5
    },
    "note": "category count only, no domain-to-version mapping in this dataset"
  },
  "scan_method": {
    "type": "passive",
    "description": "One GET request to /robots.txt only (no other path fetched), plus one TLS handshake via openssl s_client. No authentication, no vulnerability scanning, no port scanning, no exploitation attempt.",
    "user_agent": "CIAD-research/1.0",
    "http_timeout_s": 8,
    "tls_timeout_s": 8,
    "retry_policy": "one retry on timeout only, max 2 attempts per domain"
  }
}