User-agent: *
Allow: /
Disallow: /private/
Disallow: /tmp/
Disallow: /cgi-bin/

Sitemap: https://tiktokmp3.site/sitemap.xml

# Turbo Crawling Settings
Crawl-delay: 2
Request-rate: 2/1

# Allow all major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Applebot
Allow: /
Crawl-delay: 1

# Block bandwidth-heavy bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Force immediate recrawling of critical pages
User-agent: Googlebot
Request-rate: 3/1
Visit-time: 0000-0600