# Block unwanted bots
User-agent: bitlybot
Disallow: /

User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: 008
Disallow: /

User-agent: heritrix
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: NTENTbot
Disallow: /

User-agent: GigablastOpenSource
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: crawler4j
Disallow: /

User-agent: wget
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: k2spider
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: WebReaper
Disallow: /

# Allow major search engines
User-agent: Googlebot
Allow: /
Disallow: /comments/feed/
Disallow: /archives/
Disallow: /preview/
Disallow: /profiles/
Disallow: /search
Disallow: /public/

User-agent: Bingbot
Allow: /
Disallow: /comments/feed/
Disallow: /archives/
Disallow: /preview/
Disallow: /profiles/
Disallow: /search
Disallow: /public/

User-agent: Yahoo Slurp
Allow: /
Disallow: /comments/feed/
Disallow: /archives/
Disallow: /preview/
Disallow: /profiles/
Disallow: /search
Disallow: /public/

# Allow Google AdsBot for better ad performance
User-agent: Adsbot-Google
Allow: /

# Allow Google Image Bot
User-agent: Googlebot-Image
Allow: /

# Allow Google Mobile Bot
User-agent: Googlebot-Mobile
Allow: /

# Allow Mediapartners-Google (AdSense)
User-agent: Mediapartners-Google*
Allow: /

# Sitemap files
Sitemap: https://thenewsstrike.com/sitemap.xml
Sitemap: https://thenewsstrike.com/sitemap/general.xml
Sitemap: https://thenewsstrike.com/sitemap/category_site.xml
Sitemap: https://thenewsstrike.com/sitemap/news_sitemap.xml
Sitemap: https://thenewsstrike.com/sitemap/category/news-update.xml
Sitemap: https://thenewsstrike.com/sitemap/category/sports.xml
Sitemap: https://thenewsstrike.com/sitemap/category/health.xml
Sitemap: https://thenewsstrike.com/sitemap/category/business.xml
Sitemap: https://thenewsstrike.com/sitemap/category/entertainment.xml
Sitemap: https://thenewsstrike.com/sitemap/category/technology.xml
Sitemap: https://thenewsstrike.com/sitemap/category/education.xml
Sitemap: https://thenewsstrike.com/sitemap/category/law.xml
Sitemap: https://thenewsstrike.com/sitemap/category/politics.xml
Sitemap: https://thenewsstrike.com/sitemap/category/food.xml
Sitemap: https://thenewsstrike.com/sitemap/category/home.xml


# OpenAI (ChatGPT Search, User Requests, and Model Training)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic (Claude Search and Model Training)
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI (Gemini and Extended Models)
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

# Common Crawl (Used by many open-source models)
User-agent: CCBot
Allow: /