User-agent: *
Allow: /
Allow: /modules/*.css
Allow: /modules/*.gif
Allow: /modules/*.jpeg
Allow: /modules/*.jpg
Allow: /modules/*.js
Allow: /modules/*.png
Allow: /themes/*.css
Allow: /themes/*.gif
Allow: /themes/*.ico
Allow: /themes/*.jpeg
Allow: /themes/*.jpg
Allow: /themes/*.js
Allow: /themes/*.png
Allow: /themes/*.svg
Allow: /widgets/*.css
Allow: /widgets/*.js
Disallow: /admin/
Disallow: /bot/
Disallow: /greatcontent/
Disallow: /images-anc/
Disallow: /includes/
Disallow: /install/
Disallow: /modules/
Disallow: /scripts/
Disallow: /themes/
Disallow: /widgets/

# --- Google Bots ---
User-agent: AdsBot-Google
User-agent: Googlebot
Allow: /
Allow: /modules/*.css
Allow: /modules/*.gif
Allow: /modules/*.jpeg
Allow: /modules/*.jpg
Allow: /modules/*.js
Allow: /modules/*.png
Allow: /themes/*.css
Allow: /themes/*.gif
Allow: /themes/*.ico
Allow: /themes/*.jpeg
Allow: /themes/*.jpg
Allow: /themes/*.js
Allow: /themes/*.png
Allow: /themes/*.svg
Allow: /widgets/*.css
Allow: /widgets/*.js
Disallow: /admin/
Disallow: /bot/
Disallow: /greatcontent/
Disallow: /images-anc/
Disallow: /includes/
Disallow: /install/
Disallow: /modules/
Disallow: /scripts/
Disallow: /themes/
Disallow: /widgets/

# --- AI Retrieval Bots ---
User-agent: AddSearchBot                  # AddSearch (moteur de recherche interne)
User-agent: Ai2Bot                        # Allen Institute for AI (Semantic Scholar)
User-agent: AI2Bot-DeepResearchEval       # Allen Institute for AI (eval recherche)
User-agent: Ai2Bot-Dolma                  # Allen Institute for AI (dataset Dolma)
User-agent: aiHitBot                      # aiHit (analytics IA)
User-agent: amazon-kendra                 # Amazon Kendra (recherche entreprise)
User-agent: AmazonBuyForMe                # Amazon (agent d'achat IA)
User-agent: Amazonbot                     # Amazon Alexa / IA Amazon
User-agent: Amzn-SearchBot                # Amazon Search IA
User-agent: Amzn-User                     # Amazon (navigation utilisateur IA)
User-agent: Andibot                       # Andi (moteur de recherche IA)
User-agent: anthropic-ai                  # Anthropic (legacy training crawler)
User-agent: Anomura                       # Anomura AI
User-agent: ApifyBot                      # Apify (crawler generique IA)
User-agent: ApifyWebsiteContentCrawler    # Apify (scraping IA)
User-agent: Applebot                      # Apple Search / Siri
User-agent: Applebot-Extended             # Apple Intelligence / IA Apple
User-agent: Aranet-SearchBot              # Aranet (moteur de recherche IA)
User-agent: AspiegelBot                   # Huawei (crawler lie a PetalBot)
User-agent: atlassian-bot                 # Atlassian Rovo (AI search crawler)
User-agent: AwarioBot                     # Awario (veille & analyse IA)
User-agent: AwarioRssBot                  # Awario (flux RSS IA)
User-agent: AwarioSmartBot                # Awario (veille smart IA)
User-agent: AzureAI-SearchBot             # Microsoft Azure AI Search
User-agent: Baiduspider                   # Baidu (moteur de recherche/IA chinois)
User-agent: bedrockbot                    # AWS Bedrock
User-agent: bigsur.ai                     # BigSur AI
User-agent: Bravebot                      # Brave Search
User-agent: Brightbot 1.0                 # Brightbot AI
User-agent: BuddyBot                      # BuddyBot AI
User-agent: Bytespider                    # ByteDance / TikTok IA
User-agent: CCBot                         # Common Crawl (dataset open source)
User-agent: Channel3Bot                   # Channel3 AI
User-agent: ChatGLM-Spider                # Zhipu AI / ChatGLM (IA chinoise)
User-agent: ChatGPT Agent                 # OpenAI (agent autonome)
User-agent: ChatGPT-User                  # ChatGPT navigation web
User-agent: Claude-Code                   # Anthropic Claude Code
User-agent: Claude-SearchBot              # Claude (Anthropic) indexation recherche
User-agent: Claude-User                   # Claude (Anthropic) navigation web
User-agent: Claude-Web                    # Anthropic Claude (crawling web)
User-agent: ClaudeBot                     # Claude (Anthropic) entrainement
User-agent: Cloudflare-AutoRAG            # Cloudflare AutoRAG
User-agent: CloudVertexBot                # Google Cloud Vertex AI (variante)
User-agent: Code                          # GitHub Copilot (agent de code IA)
User-agent: cohere-ai                     # Cohere (LLM canadien)
User-agent: cohere-training-data-crawler  # Cohere (entrainement)
User-agent: Cotoyogi                      # Cotoyogi AI
User-agent: Crawl4AI                      # Crawl4AI (crawler open source)
User-agent: Crawlspace                    # Crawlspace AI
User-agent: Datenbank Crawler             # Datenbank (base de donnees IA)
User-agent: DataForSeoBot                 # DataForSEO (donnees SEO IA)
User-agent: DeepSeekBot                   # DeepSeek (IA chinoise)
User-agent: Devin                         # Cognition AI (agent dev)
User-agent: DiffBot                       # Diffbot (extraction de donnees IA)
User-agent: DigitalOceanGenAICrawler      # DigitalOcean GenAI
User-agent: DuckAssistBot                 # DuckDuckGo AI
User-agent: Echobot Bot                   # Echobot (veille medias IA)
User-agent: EchoboxBot                    # Echobox (social media IA)
User-agent: ExaBot                        # Exa AI (recherche IA)
User-agent: FacebookBot                   # Meta / Facebook crawler IA
User-agent: facebookexternalhit           # Meta / apercu liens Facebook
User-agent: Factset_spyderbot             # FactSet (donnees financieres IA)
User-agent: FirecrawlAgent                # Firecrawl (scraping IA)
User-agent: FriendlyCrawler               # Friendly AI
User-agent: Gemini-Deep-Research          # Google Gemini (recherche approfondie)
User-agent: Google-Agent                  # Google Agent IA (variante)
User-agent: Google-CloudVertexBot         # Google Cloud Vertex AI
User-agent: Google-Extended               # Google Gemini (entrainement IA)
User-agent: Google-Firebase               # Google Firebase AI
User-agent: Google-Gemini-CLI             # Google Gemini CLI (agent de code IA)
User-agent: Google-NotebookLM             # Google NotebookLM
User-agent: GoogleAgent-Mariner           # Google Agent (navigation web)
User-agent: GoogleAgent-URLContext        # Google Agent (contexte URL IA)
User-agent: GoogleOther                   # Google autres usages IA
User-agent: GoogleOther-Image             # Google images pour IA
User-agent: GoogleOther-Video             # Google video pour IA
User-agent: GPTBot                        # OpenAI (entrainement GPT)
User-agent: Grok                          # xAI Grok (Live Search)
User-agent: Grok-DeepSearch               # xAI Grok (recherche approfondie)
User-agent: GrokBot                       # xAI / Grok
User-agent: iAskBot                       # iAsk.ai (variante)
User-agent: iaskspider                    # iAsk.ai (moteur de recherche IA)
User-agent: IbouBot                       # Ibou AI
User-agent: ICC-Crawler                   # ICC (analyse de contenu)
User-agent: imageSpider                   # Image Spider (collecte images IA)
User-agent: ImagesiftBot                  # ImageSift (classification images IA)
User-agent: img2dataset                   # LAION (dataset images open source)
User-agent: kagi-fetcher                  # Kagi (moteur de recherche)
User-agent: Kangaroo Bot                  # Kangaroo LLM
User-agent: KlaviyoAIBot                  # Klaviyo AI (marketing)
User-agent: KomoBot                       # Komo AI (moteur de recherche IA)
User-agent: KunatoCrawler                 # Kunato AI
User-agent: laion-huggingface-processor   # LAION/HuggingFace (traitement datasets)
User-agent: LAIONDownloader               # LAION (telechargement datasets)
User-agent: LCC                           # LCC AI
User-agent: Lightpanda                    # Lightpanda (navigateur headless pour agents IA)
User-agent: LinerBot                      # Liner AI
User-agent: Linguee Bot                   # Linguee/DeepL (traduction IA)
User-agent: LinkupBot                     # Linkup AI
User-agent: magpie-crawler                # Magpie (collecte donnees entrainement IA)
User-agent: Manus-User                    # Manus AI (agent autonome)
User-agent: Meltwater                     # Meltwater (media intelligence IA)
User-agent: Meta-ExternalAgent            # Meta (agent externe IA majuscule)
User-agent: Meta-ExternalFetcher          # Meta (reponses IA temps reel)
User-agent: meta-externalagent            # Meta (agent externe IA minuscule)
User-agent: meta-externalfetcher          # Meta (fetcher IA minuscule)
User-agent: meta-webindexer               # Meta (indexation web IA)
User-agent: MistralAI-User                # Mistral AI
User-agent: MyCentralAIScraperBot         # MyCentral AI
User-agent: NewsBreakBot                  # NewsBreak (actualites IA)
User-agent: NotebookLM                    # Google NotebookLM (variante)
User-agent: NovaAct                       # Amazon Nova Act (agent IA)
User-agent: OAI-SearchBot                 # OpenAI Search (recherche web)
User-agent: omgili                        # Webz.io / Omgili (crawler news IA)
User-agent: omgilibot                     # Webz.io / Omgili (variante)
User-agent: OpenAI                        # OpenAI (generique)
User-agent: opencode                      # opencode (agent de code IA open source)
User-agent: Operator                      # OpenAI Operator (agent web)
User-agent: PanguBot                      # Huawei PanGu (IA chinoise)
User-agent: Panscient                     # Panscient AI
User-agent: Panscient.com                 # Panscient AI (variante domaine)
User-agent: peer39_crawler                # Peer39 (ciblage contextuel IA)
User-agent: Perplexity-User               # Perplexity AI (navigation web temps reel)
User-agent: PerplexityBot                 # Perplexity AI (recherche IA)
User-agent: PetalBot                      # Huawei Petal Search
User-agent: PhindBot                      # Phind (recherche IA pour devs)
User-agent: PoeBot                        # Poe/Quora (plateforme chatbot IA)
User-agent: PoeSearchBot                  # Poe/Quora (recherche IA)
User-agent: Poggio-Citations              # Poggio (citations IA)
User-agent: Poseidon Research Crawler     # Poseidon (recherche IA)
User-agent: QualifiedBot                  # Qualified (marketing IA)
User-agent: QuillBot                      # QuillBot (paraphrase IA)
User-agent: SBIntuitionsBot               # SB Intuitions (IA japonaise)
User-agent: Seekr                         # Seekr (evaluation contenu IA variante)
User-agent: SeekrBot                      # Seekr (evaluation contenu IA)
User-agent: SemrushBot-OCOB               # Semrush ContentShake AI (optimisation contenu GEO)
User-agent: Sentibot                      # Sentibot (analyse de sentiment IA)
User-agent: ShapBot                       # ShapBot AI
User-agent: Sidetrade indexer bot         # Sidetrade (IA financiere)
User-agent: Sogou web spider              # Sogou (moteur de recherche/IA chinois)
User-agent: Spider                        # Spider.cloud (scraping IA)
User-agent: TavilyBot                     # Tavily (API recherche IA)
User-agent: TerraCotta                    # TerraCotta AI
User-agent: Thinkbot                      # Thinkbot AI
User-agent: TikTokSpider                  # TikTok (crawler IA)
User-agent: Timpibot                      # Timpi (moteur de recherche decentralise)
User-agent: Trae                          # ByteDance Trae (IDE/agent de code IA)
User-agent: TurnitinBot                   # Turnitin (detection contenu IA)
User-agent: TwinAgent                     # TwinAgent AI
User-agent: VelenPublicWebCrawler         # Velen AI
User-agent: WARDBot                       # WARD AI (securite IA)
User-agent: webzio-extended               # Webz.io (variante minuscule)
User-agent: Webzio-Extended               # Webz.io (donnees web pour IA)
User-agent: wpbot                         # WPBot (WordPress IA)
User-agent: WRTNBot                       # WRTN (IA coreenne)
User-agent: xAI-Grok                      # xAI Grok (variante user-agent)
User-agent: YaK                           # YaK AI
User-agent: YandexAdditional              # Yandex IA
User-agent: YandexAdditionalBot           # Yandex IA (variante)
User-agent: YisouSpider                   # Yisou (moteur de recherche chinois)
User-agent: YouBot                        # You.com (recherche IA)
User-agent: ZanistaBot                    # Zanista AI
Allow: /
Allow: /modules/*.css
Allow: /modules/*.gif
Allow: /modules/*.jpeg
Allow: /modules/*.jpg
Allow: /modules/*.js
Allow: /modules/*.png
Allow: /themes/*.css
Allow: /themes/*.gif
Allow: /themes/*.ico
Allow: /themes/*.jpeg
Allow: /themes/*.jpg
Allow: /themes/*.js
Allow: /themes/*.png
Allow: /themes/*.svg
Allow: /widgets/*.css
Allow: /widgets/*.js
Disallow: /admin/
Disallow: /bot/
Disallow: /greatcontent/
Disallow: /images-anc/
Disallow: /includes/
Disallow: /install/
Disallow: /modules/
Disallow: /scripts/
Disallow: /themes/
Disallow: /widgets/

Sitemap: https://www.creperie-sarrasin.fr/sitemap.xml