# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# Most AI crawlers willfully ignore the intent of the CC-BY license.
#
# For questions about this, send
# email to Kevin McCurley
#
# Note that we check the list of approved IP addresses for Googlebot
User-agent: Googlebot
Disallow: /archive
Disallow: /versions
Disallow: /submit
Disallow: /search
# they get it from other places with abstracts for better indexing
Disallow: /complete/compact
Allow: *
# ################################ bing
User-agent: bingbot
Crawl-delay: 5
Disallow: *pdf
Disallow: /archive
Disallow: /versions
Disallow: /submit
Disallow: /search
# they get it from other places with abstracts for better indexing
Disallow: /complete/compact
Allow: *
# duckduckgo
# ################################ duckduckgo
User-agent: DuckDuckBot/1.1
Crawl-delay: 5
Disallow: *pdf
Disallow: /archive
Disallow: /versions
Disallow: /submit
Disallow: /search
# they get it from other places with abstracts for better indexing
Disallow: /complete/compact
Allow: *
##########################################
# AI bots get limited access. No PDFs.
User-agent: Googleextended
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: GPTBot
# claude uses 160.79.104.0/21 and 160.79.104.0/23 and 2607:6bc0::/48
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
Disallow: *.pdf
Disallow: /199*/*.pdf
Disallow: /200*/*.pdf
Disallow: /201*/*.pdf
Disallow: /202*/*.pdf
Allow: /199*/*
Allow: /200*/*
Allow: /201*/*
Allow: /202*/*
Disallow: /*/*pdf
Disallow: /archive
Disallow: /versions
Disallow: /submit
Disallow: /search
Disallow: *
# ################################ everyone else. Crawlers must be whitelisted and
# confirmed to be following rules, which many do not.
User-agent: *
Disallow: /