# Broken robots.txt — every tech present but implemented wrong. # robotsTxtValid=false, perBotRuleValid=false, contentSignalsValid=false. # robotsTxtDirectiveBeforeGroup: a Disallow before any User-agent group. Disallow: /private/ # robotsTxtEmptyUserAgent: User-agent with no value. User-agent: Disallow: /nowhere/ User-agent: * # contentSignalUnknownValue (ai-train=maybe) + contentSignalEmptyKeyValue (=no). Content-Signal: search=yes, ai-train=maybe, =no # robotsTxtMissingColon: no colon separator. Allow / # robotsTxtUnknownDirective. Wibble: nonsense # perBotRuleMisspelledToken (GPT-Bot vs GPTBot) + perBotRuleContradictoryAllowDisallow (/). User-agent: GPT-Bot Allow: / Disallow: / # perBotRuleDuplicateGroup: CCBot defined in two separate groups. User-agent: CCBot Disallow: /tmp/ User-agent: CCBot Disallow: /docs/ # perBotRuleEmptyGroup: Bytespider group with no rules. User-agent: Bytespider