# Robots.txt for avestalabs.ai
# Product engineering consultancy specializing in MVP development and engineering excellence
#
# NOTE ON GROUPS: a crawler obeys only the single most specific User-agent
# group that matches it, and ignores every other group. A named group with
# just "Allow: /" therefore cancels all the Disallow rules below for that
# crawler. Any named group that needs the site-wide blocks must repeat them.

User-agent: *
Allow: /

# Block crawling of API endpoints and administrative areas.
# /_next/ is deliberately NOT blocked, in any group: it holds the JS and CSS
# crawlers need to render pages, and /_next/image serves every blog cover
# and article image. Blocking it hid those images from Google Images and from
# AI crawlers, and made rendered pages look broken to Googlebot.
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /draft/
Disallow: /preview/
Disallow: /agent-demo/

# Block common files that shouldn't be indexed
Disallow: /*.json$
Disallow: /sitemap.xml.gz

# Self-hosted media is decorative (hero background loops and product UI
# screencasts), never the main content of a page. This rule only saves
# crawl budget on ~45 MB of media -- it does NOT clear the "Video isn't on
# a watch page" report, as was assumed when it was added. Google discovers
# those videos from the <video>/<source> tags in the served HTML, not by
# crawling this directory, so the report stays put for as long as the
# elements ship. Removing the element is the only thing that clears it,
# and we have decided the decorative loops are worth the note.
Disallow: /videos/

# Allow specific important files
Allow: /favicon.ico
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt

# Crawl delay (helps with server load)
Crawl-delay: 1

# Sitemap location
Sitemap: https://avestalabs.ai/sitemap.xml

# Specific rules for major search engines.
# Googlebot deliberately has no group of its own so that it inherits the
# rules above. Google ignores Crawl-delay, so a Googlebot group could only
# ever weaken this file.

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /draft/
Disallow: /preview/
Disallow: /agent-demo/
Disallow: /*.json$
Disallow: /sitemap.xml.gz
Disallow: /videos/
Crawl-delay: 1

# Social unfurlers fetch only the specific URL a user shared, so they are
# intentionally unrestricted.
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# Allow AI/LLM crawlers for better AI understanding, minus the same
# non-content areas blocked above.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /draft/
Disallow: /preview/
Disallow: /agent-demo/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /draft/
Disallow: /preview/
Disallow: /agent-demo/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /draft/
Disallow: /preview/
Disallow: /agent-demo/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/