# unt24.by — robots.txt
# Карта сайта для LLM-ассистентов: https://unt24.by/llms.txt
# XML-карта для классических краулеров: https://unt24.by/sitemap.xml
#
# Структура файла:
#   1. AI-краулеры — явное согласие на индексирование
#   2. Все остальные боты (User-agent: *)
#
# Последнее обновление: 2026-07-24 (Disallow /include/ — служебные фрагменты шаблона)


# ============================================================
# 1. AI-краулеры — явный allowlist
# ============================================================
# OpenAI
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai

# Google (Gemini / Bard — не путать с обычным Googlebot)
User-agent: Google-Extended

# Apple Intelligence
User-agent: Applebot-Extended

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User

# Yandex GPT / Алиса
User-agent: YandexAdditional
User-agent: YandexAdditionalBot

# Common Crawl — обучающая база для десятков моделей
User-agent: CCBot

# Meta (Llama / AI-ассистенты)
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher

# ByteDance (Doubao, TikTok AI)
User-agent: Bytespider

# Cohere
User-agent: cohere-ai

# Diffbot (DiffbotGPT, knowledge graph)
User-agent: Diffbot

# Amazon (Alexa+, Amazon Q)
User-agent: Amazonbot

# DuckDuckGo AI (DuckAssist)
User-agent: DuckAssistBot

# Mistral
User-agent: MistralAI-User

Allow: /
Disallow: /bitrix/
Disallow: /personal/
Disallow: /include/
Disallow: /*login=
Disallow: /*logout=
Disallow: /*auth=
Disallow: /*register=
Disallow: /*forgot_password=
Disallow: /*change_password=
Disallow: /*backurl=
Disallow: /*back_url=
Disallow: /*BACKURL=
Disallow: /*BACK_URL=
Disallow: /*?action=
Disallow: /*&action=
Disallow: /*?print=
Disallow: /*&print=
Disallow: /*clear_cache=
Disallow: /*show_sql_stat=
Disallow: /*ADD_TO_COMPARE_LIST
Disallow: /*ORDER_BY
Disallow: /*PAGEN
Allow: /bitrix/components/
Allow: /bitrix/js/
Allow: /bitrix/templates/


# ============================================================
# 2. Все остальные краулеры
# ============================================================
User-agent: *
Disallow: /bitrix/
Disallow: /personal/
Disallow: /include/
Disallow: /*show_include_exec_time=
Disallow: /*show_page_exec_time=
Disallow: /*show_sql_stat=
Disallow: /*bitrix_include_areas=
Disallow: /*clear_cache=
Disallow: /*clear_cache_session=
Disallow: /*ADD_TO_COMPARE_LIST
Disallow: /*ORDER_BY
Disallow: /*PAGEN
Disallow: /*?print=
Disallow: /*&print=
Disallow: /*print_course=
Disallow: /*?action=
Disallow: /*&action=
Disallow: /*register=
Disallow: /*forgot_password=
Disallow: /*change_password=
Disallow: /*login=
Disallow: /*logout=
Disallow: /*auth=
Disallow: /*backurl=
Disallow: /*back_url=
Disallow: /*BACKURL=
Disallow: /*BACK_URL=
Disallow: /*back_url_admin=
Disallow: /*?bxajaxid=
Disallow: /*&bxajaxid=

# Маркетинговые параметры — устранение дублей в индексе
Disallow: /*?utm_source=
Disallow: /*&utm_source=
Disallow: /*?utm_medium=
Disallow: /*&utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*&utm_campaign=
Disallow: /*?utm_term=
Disallow: /*&utm_term=
Disallow: /*?utm_content=
Disallow: /*&utm_content=
Disallow: /*?gclid=
Disallow: /*&gclid=
Disallow: /*?fbclid=
Disallow: /*&fbclid=
Disallow: /*?yclid=
Disallow: /*&yclid=
Disallow: /*?ymclid=
Disallow: /*&ymclid=
Disallow: /*?_openstat=
Disallow: /*&_openstat=

# Битрикс-ассеты разрешены, чтобы краулеры рендерили страницы корректно
Allow: /bitrix/components/
Allow: /bitrix/js/
Allow: /bitrix/templates/

Sitemap: https://unt24.by/sitemap.xml