# robots.txt für yoice.net
# Letzte Aktualisierung: Juli 2025

# Standard-Regeln für alle Crawler
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /wp-config.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /my-account/
Disallow: /search/
Disallow: /?s=
Disallow: /feed/
Disallow: /?author=*
Disallow: /comments/feed/
Disallow: /wp-json/
Disallow: */trackback/
Disallow: */preview/
Disallow: */?replytocom=
Disallow: */attachment/
Disallow: /tag/
Disallow: /category/
Disallow: /*?s=
# Uploads erlauben für Medien-SEO (Bilder und Audio in normaler Sitemap via RankMath)
Allow: /wp-content/uploads/*.jpg
Allow: /wp-content/uploads/*.jpeg
Allow: /wp-content/uploads/*.png
Allow: /wp-content/uploads/*.webp
Allow: /wp-content/uploads/*.gif
Allow: /wp-content/uploads/*.svg
Allow: /wp-content/uploads/*.mp3
Allow: /wp-content/uploads/*.wav
Allow: /wp-content/uploads/*.m4a
Disallow: /wp-content/uploads/*.php
Disallow: /wp-content/uploads/*.txt

# WordPress spezifische Dateien
Disallow: /readme.html
Disallow: /license.txt

# Temporäre oder interne Bereiche
Disallow: /temp/
Disallow: /tmp/
Disallow: /private/
Disallow: /backup/

# Crawling-Verzögerung für aggressive Bots
Crawl-delay: 10

# Explizite Erlaubnis für wichtige Suchmaschinen-Bots
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: Slurp
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Crawl-delay: 5

User-agent: YandexBot
Allow: /
Crawl-delay: 3

# Blockiere aggressive SEO-Tools und Scraper
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Erlaube Zugriff für OpenAI, ChatGPT und Claude (wichtig für KI-SEO)
User-agent: GPTBot
Allow: /
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 5

User-agent: Claude-Web
Allow: /
Crawl-delay: 5

User-agent: CCBot
Allow: /
Crawl-delay: 10

# Social Media Bots (falls nicht gewünscht)
User-agent: facebookexternalhit
Disallow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Sitemap-Referenz
Sitemap: https://yoice.net/sitemap_index.xml