# Machine-readable licensing terms (RSL 1.0 — https://rslstandard.org/rsl).
# The formal twin of the allow/deny split below: robots.txt can only say ALLOW or
# DENY, never "allowed, but under these terms" — which is the actual policy here
# (search/citation free with attribution, AI training by paid licence only, see
# /vilkaar §4-5 and §7). Keep in sync with client/public/.well-known/rsl.xml.
#
# MUST stay ABOVE the first `User-agent:` line. RSL 1.0 §4.4 scopes License the
# way RFC 9309 scopes group members: a directive after a User-agent line belongs
# to THAT group, and only a global directive (outside every group) reaches all
# clients. Placed at the bottom it sat inside the last group — ImagesiftBot,
# which is Disallow: / — so the one crawler that could see it was the one
# crawler told not to read anything, and every other bot found no terms at all.
# Blank lines and comments do NOT terminate a group; only the next User-agent
# does. Do not "tidy" this down beside Sitemap: — Sitemap is an RFC 9309
# non-group field and may live anywhere, License is not.
#
# Lighthouse / PageSpeed Insights flags this line "Unknown directive" and fails
# its robots.txt audit (~8 SEO points). Known and accepted 2026-10-03: Google's
# parser ignores unknown lines, and the fix is Lighthouse's to ship
# (GoogleChrome/lighthouse#17160 adds `license` to its safelist). Do not delete
# this line to turn that audit green.
License: https://bildata.io/.well-known/rsl.xml

# Allow all crawlers (default)
User-agent: *
Allow: /
Disallow: /debug

# === Search / inference AI bots — ALLOWED ===
# Free citation and search use is permitted under our terms (see /vilkaar).
# These bots crawl for real-time search retrieval and in-chat browsing — not for model training.

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

# === Ambiguous (used for both training and inference) — currently ALLOWED ===
# ClaudeBot powers both Claude training crawls AND live Claude web features.
# Anthropic does not separate these into different bots. Blocking it would
# lose Claude citation entirely. Keeping allowed; legal opt-out via /vilkaar.
User-agent: ClaudeBot
Allow: /

# === Training-only AI bots — BLOCKED ===
# Use of Bildata.io content for AI / ML model training is prohibited without
# a separate paid license (see /vilkaar §4-5). Contact info@bildata.io for
# commercial licensing.

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Sitemap location
Sitemap: https://bildata.io/sitemap.xml