User-agent: *
Allow: /
Disallow: /admin/
Disallow: /includes/
Disallow: /config/
Disallow: /cache/
Disallow: /logs/
Disallow: /cron/
Disallow: /api/
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.lock$

# 屏蔽Facebook蜘蛛
User-agent: facebookexternalhit
Disallow: /
User-agent: facebookcatalog
Disallow: /
User-agent: Facebot
Disallow: /

# 屏蔽Amazon蜘蛛
User-agent: Amazonbot
Disallow: /
User-agent: Amazonbot/1.0
Disallow: /
User-agent: Amazonbot/2.0
Disallow: /

# 屏蔽Semrush蜘蛛
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /

# 允许搜索引擎访问重要页面
Allow: /sitemap.xml
Allow: /sitemap-index.xml
Allow: /robots.txt
Allow: /rss/

# 设置爬取延迟
Crawl-delay: 1

# Sitemap位置
Sitemap: https://yourdomain.com/sitemap.xml
Sitemap: https://yourdomain.com/sitemap-index.xml