# 1Lookup, https://www.1lookup.io
# Default: allow all traditional search crawlers.
User-agent: *
Allow: /
# The LLM index files, the OpenAPI spec and the MCP server card are meant to
# be fetched.
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.json
Allow: /api/openapi.yaml
Allow: /.well-known/mcp/server-card.json

# --- AI / answer-engine crawlers (explicitly allowed) ---
# 1Lookup wants to be discoverable and citable in AI answers, so the major
# AI training and AI-search user-agents are explicitly welcomed. See
# /llms.txt and /llms-full.txt for a structured index of the site.

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI (Gemini / Vertex training), separate from Googlebot search
User-agent: Google-Extended
Allow: /

# Google grounding for Vertex AI, separate from Google-Extended
User-agent: Google-CloudVertexBot
Allow: /

# Apple. Applebot drives Siri and Spotlight answers; Applebot-Extended governs
# training only, so both are named.
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI (training crawler, and the user-triggered fetch that produces citations)
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Amazon (feeds Alexa and Rufus answers)
User-agent: Amazonbot
Allow: /

# DuckDuckGo AI answers
User-agent: DuckAssistBot
Allow: /

# Mistral Le Chat, user-triggered fetch
User-agent: MistralAI-User
Allow: /

# Cohere grounded generation
User-agent: cohere-ai
Allow: /

# You.com answer engine
User-agent: YouBot
Allow: /

# Allen Institute for AI (OLMo corpora, feeds open models)
User-agent: Ai2Bot
Allow: /

# Common Crawl (feeds many downstream LLMs)
User-agent: CCBot
Allow: /

# Sitemap location
Sitemap: https://www.1lookup.io/sitemap.xml
# The indexable slice of the ZIP code cluster gets its own file. The other
# ~40,500 ZIP pages stay live and linked but render noindex, so they are not
# listed in any sitemap.
Sitemap: https://www.1lookup.io/zip-sitemap.xml

# LLM-readable index
# https://www.1lookup.io/llms.txt
# https://www.1lookup.io/llms-full.txt

# MCP server card (machine-readable connection details for AI agents)
# https://www.1lookup.io/.well-known/mcp/server-card.json

# OpenAPI description of the 1Lookup REST API
# https://www.1lookup.io/openapi.json
# https://www.1lookup.io/api/openapi.yaml

# Markdown content negotiation (https://acceptmarkdown.com)
# Every page also answers `Accept: text/markdown` with a markdown
# representation of itself, and varies on Accept.

# Disallow admin/private paths if any
# Disallow: /admin/
# Disallow: /api/