# robots.txt for dev.calcolifiscali.com
# Generated: 2026-01-29T10:36:59Z
# 2025 AI Crawler Standards

User-agent: *
Allow: /

# Block utility pages (uncomment if needed)
# Disallow: /search/
# Disallow: /filter/
# Disallow: /preview/

# ========================================================================
# AI Model Training Crawlers (explicitly allowed for public content)
# ========================================================================

# OpenAI GPT models
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic Claude models
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# ByteDance AI crawlers
User-agent: Bytespider
Allow: /

# Meta AI training
User-agent: Meta-ExternalAgent
Allow: /

# Apple AI crawlers
User-agent: Applebot-Extended
Allow: /

# Common Crawl archive
User-agent: CCBot
Allow: /

# DuckDuckGo AI assistant
User-agent: DuckAssistant
Allow: /

# Google AI training
User-agent: Google-Extended
Allow: /

# ========================================================================
# Sitemap & Feeds
# ========================================================================

Sitemap: https://dev.calcolifiscali.com/sitemap.xml