# --- Block known AI training/usage agents ---
User-agent: 008
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Bytedance
Disallow: /

# --- General rules (block internal, gated, or obsolete directories) ---
User-agent: *
Disallow: /clients/
Disallow: /community/casebrief/b/
Disallow: /community/members/
Disallow: /em/
Disallow: /en-us/campaign/
Disallow: /en-us/ppc/
Disallow: /en-us/products/test/
Disallow: /en-us/test/
Disallow: /en-us/demo/
Disallow: /internationalorderform/
Disallow: /LegalNewsRoom/members/
Disallow: /lextalk/members/
Disallow: /martindale-hubbell/
Disallow: /pdf/practical-guidance/journal/
Disallow: /portal/
Disallow: /preferencecenter/
Disallow: /supp/LargeLaw/no-index
Disallow: /supp/lawschool/class/publications/pdf/
Disallow: /supp/lawschool/LAReports/
Disallow: /supp/lawschool/resources/no-index
Disallow: /supp/lawschool/texts/pdf/
Disallow: /supp/lawschool/Tutorials/
Disallow: /supp/lawschool/Zaption/
Disallow: /support/
Disallow: /test/

# --- AUNZ Disallow search (DO NOT DELETE) ---
Disallow: /assets/en-au/pdf/Lexis-Plus-AI-Protege-General-AI-Model-Comparison.pdf
Disallow: /assets/en-au/pdf/Lexis-Create-Plus-FAQs.pdf
Disallow: /assets/en-au/pdf/Lexis-Create-Plus-InstallGuide-2025.pdf
Disallow: /assets/en-au/pdf/Lexis-Create-Plus-System-Requirements.pdf
Disallow: /assets/en-au/pdf/Lexis-Create-Plus-User-Guide-2025.pdf
Disallow: /assets/en-nz/pdf/Lexis_Plus_AI_and_Lexis_Plus_Integrated_Ecosystem_202409.pdf
Disallow: /assets/en-nz/pdf/Lexis_Plus_AI_NZ_Prompting_Framework_Guide_v0_1_29_04.pdf
Disallow: /assets/en-nz/pdf/LexisNexisProtege-Info-Security-06252025-NZ-V4-JUNE-FINAL.pdf
Disallow: /assets/en-au/pdf/Protege-Guide-Workflow-Powered-AI.pdf
Disallow: /assets/en-au/pdf/Protege-Guide-Transactional-Workflows.pdf
Disallow: /assets/en-au/pdf/Protege-Guide-Litigation-Workflows.pdf
Disallow: /assets/en-au/pdf/Protege-Guide-General-Workflows.pdf
Disallow: /assets/en-au/pdf/Protege-Guide-Simplified-Interface.pdf
Disallow: /assets/en-au/pdf/Custom-Workflow-Builder-Guide.pdf
Disallow: /assets/en-nz/pdf/general-workflows-nz.pdf
Disallow: /assets/en-nz/pdf/litigation-workflow-nz.pdf
Disallow: /assets/en-nz/pdf/lexis-plus-with-protégé-workflow-powered-legal-ai-for-every-role-nz.pdf
Disallow: /assets/en-nz/pdf/lexis-plus-with-protégé-simplified-ui-foundationals-full-version-nz.pdf
Disallow: /assets/en-nz/pdf/custom-workflow-builder-guide-us-version.pdf
Disallow: /assets/en-au/pdf/protege-work-skills-quick-start-guide.pdf
Disallow: /assets/en-au/pdf/protege-skills-user-guide.pdf

# --- GNS Disallow search (DO NOT DELETE) ---
Disallow: /pdf/global/LNGlobal-Agreement-gNS-Data-Benelux-and-Channels-21.pdf
# --- Disallow search and parameterized URLs ---
# Prevents crawl duplication and infinite loops, but does NOT affect GA4 tracking.
Disallow: /*?*
Disallow: /*&*
Disallow: /search
Disallow: /search?

# --- Blog-specific crawl management (added 2025-11) ---
# Blocks dynamic, non-indexable endpoints and low-value tag feeds.
Disallow: /blogs/api.ashx/
Disallow: /blogs/tags/
Disallow: /blogs/*/tags/
Disallow: /*.axd$
Disallow: /legalnewsroom
Disallow: /lextalk

# --- Disallow tolley ln.com pages (added 3/25/2026) ---
Disallow: /authorcenter/experts
Disallow: /community/knowledge-centre

# --- AI AGENTS / LLM CRAWLERS --- For EN-GB/LEGAL ---
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: GPTBot
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Perplexity-ai
User-agent: PerplexityBot
User-agent: MetaScraper
User-agent: FacebookBot
User-agent: facebookexternalhit
User-agent: Applebot-Extended
User-agent: Applebot
User-agent: Amazonbot
User-agent: Bytedance
User-agent: Bytespider
User-agent: CohereBot
User-agent: cohere-ai
User-agent: YouBot
User-agent: You.com Bot
User-agent: CCBot
User-agent: aiHitBot

# --- NON-AI CRAWLERS (search engines, archives, SEO, data tools)
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
User-agent: DuckDuckBot
User-agent: SeznamBot
User-agent: PetalBot
User-agent: Sogou Spider
User-agent: Sogou web spider
User-agent: BLEXbot
User-agent: Barkrowler
User-agent: BDCbot
User-agent: heritrix
User-agent: ia_archiver
User-agent: omgili
User-agent: omgilibot
User-agent: PiplBot
User-agent: magpie-crawler
User-agent: NewsNow
User-agent: news-please
User-agent: bitlybot
User-agent: Sirius
User-agent: Neevabot
Disallow: /en-gb/legal/

# --- Sitemap index ---
Sitemap: https://www.lexisnexis.com/sitemap__index.xml