# ============================================================
# robots.txt — Netsoft (www.netsoft.com)
# Última actualización: 2026
# Objetivo: Permitir indexación completa a motores de búsqueda
# y bots de IA. Proteger rutas administrativas y privadas.
# ============================================================


# ------------------------------------------------------------
# BOTS DE IA — ACCESO TOTAL (crítico para GEO / LLM citation)
# ------------------------------------------------------------

# OpenAI — ChatGPT
User-agent: GPTBot
Allow: /

# OpenAI — ChatGPT browsing
User-agent: ChatGPT-User
Allow: /

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google — Gemini y AI Overviews
User-agent: Google-Extended
Allow: /

# Microsoft — Copilot (vía Bing)
User-agent: Bingbot
Allow: /

# Meta — Llama / AI features
User-agent: FacebookBot
Allow: /


# ------------------------------------------------------------
# MOTORES DE BÚSQUEDA PRINCIPALES — ACCESO TOTAL
# ------------------------------------------------------------

User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /admin/
Disallow: /login/
Disallow: /dashboard/
Disallow: /portal/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /*.env$
Disallow: /*.log$
Disallow: /xmlrpc.php

User-agent: Slurp
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /admin/
Disallow: /login/

User-agent: DuckDuckBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /admin/
Disallow: /login/


# ------------------------------------------------------------
# BOTS DE SCRAPING Y ENTRENAMIENTO MASIVO — BLOQUEADOS
# Estos no aportan citación ni tráfico. Solo consumen recursos.
# ------------------------------------------------------------

# Common Crawl (usado para entrenar modelos, no para citación)
User-agent: CCBot
Disallow: /

# Scrapers genéricos conocidos
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /


# ------------------------------------------------------------
# REGLA GENERAL — todos los demás bots no listados
# ------------------------------------------------------------

User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /admin/
Disallow: /login/
Disallow: /dashboard/
Disallow: /portal/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /*.env$
Disallow: /*.log$
Disallow: /xmlrpc.php


# ------------------------------------------------------------
# SITEMAP
# ------------------------------------------------------------

Sitemap: https://www.netsoft.com/sitemap.xml