User-agent: * Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /73649979575/checkouts Disallow: /73649979575/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://safegirl.co.il/sitemap.xml User-agent: adsbot-google Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /orders Disallow: /73649979575/checkouts Disallow: /73649979575/orders Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /cdn/wpm/*.js Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /services/login_with_shop User-agent: Nutch Disallow: / User-agent: AhrefsBot Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /73649979575/checkouts Disallow: /73649979575/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://safegirl.co.il/sitemap.xml User-agent: AhrefsSiteAudit Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /73649979575/checkouts Disallow: /73649979575/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://safegirl.co.il/sitemap.xml User-agent: MJ12bot Crawl-delay: 10 User-agent: Pinterest Crawl-delay: 1 # ──────────────────────────────────────────────────────────────── # AI crawlers — explicit access for accurate brand representation # בוטי AI - גישה מפורשת לייצוג מדויק של המותג SafeGirl # ──────────────────────────────────────────────────────────────── # OpenAI — used to train GPT models User-agent: GPTBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # OpenAI — fetches pages on behalf of ChatGPT users (live browsing) User-agent: ChatGPT-User Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # OpenAI — SearchGPT index crawler User-agent: OAI-SearchBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Google — Bard, Gemini, AI Overviews training/grounding User-agent: Google-Extended Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Google — Vertex AI / search snippets User-agent: GoogleOther Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Anthropic — Claude training crawler User-agent: ClaudeBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Anthropic — legacy identifier User-agent: anthropic-ai Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Anthropic — Claude live browsing User-agent: Claude-Web Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Anthropic — newer Claude user-fetch agent User-agent: Claude-User Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Perplexity AI — index crawler User-agent: PerplexityBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Perplexity — live user fetch User-agent: Perplexity-User Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Apple Intelligence — Siri / Apple Intelligence model training User-agent: Applebot-Extended Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Common Crawl — feeds many open-source AI training datasets User-agent: CCBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Meta AI — public web crawler used by Meta AI / Llama products User-agent: Meta-ExternalAgent Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Meta AI — Facebook AI assistants live fetch User-agent: FacebookBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Mistral AI User-agent: MistralAI-User Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Cohere AI User-agent: cohere-ai Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # DuckDuckGo AI assistant User-agent: DuckAssistBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # You.com User-agent: YouBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Diffbot — knowledge graph used by AI applications User-agent: Diffbot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # ──────────────────────────────────────────────────────────────── # Honeypot — DO NOT VISIT. # This path is a bot-trap. Any well-behaved crawler MUST honor this # disallow. Anything that fetches /pages/sg-trap-do-not-visit despite # this rule is, by definition, a misbehaving scraper. The page itself # logs the visitor and triggers an alert. # ──────────────────────────────────────────────────────────────── User-agent: * Disallow: /pages/sg-trap-do-not-visit # Pointer to AI-friendly site descriptions # /llms.txt # /llms-full.txt # ──────────────────────────────────────────────────────────────── # Social / ad / commerce crawlers — EXPLICITLY ALLOWED # חשוב: בוטים אלה הכרחיים להופעת המוצרים שלנו במודעות Meta/Google, # בתצוגות שיתוף בוואטסאפ/טלגרם/אינסטגרם, ובקטלוגים של פייסבוק. # חסימה שלהם = מודעות בלי תמונה + לינקים בלי תיאור + ירידה ב-CTR. # ──────────────────────────────────────────────────────────────── # Facebook / Instagram / WhatsApp link previews + Open Graph # (משמש את כל מודעות Meta, שיתופים, Instagram Shopping, FB Catalog) User-agent: facebookexternalhit Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Facebook crawler (legacy) User-agent: Facebot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Instagram crawler User-agent: Instagram Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Twitter / X link previews User-agent: Twitterbot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # LinkedIn link previews User-agent: LinkedInBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Pinterest (rich pins, shopping) User-agent: Pinterest Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders User-agent: Pinterestbot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # WhatsApp link previews User-agent: WhatsApp Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Telegram link previews User-agent: TelegramBot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Slack link previews (if SafeGirl ever shares deals in Slack communities) User-agent: Slackbot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Google Shopping / Merchant Center / Ads User-agent: Googlebot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders User-agent: Googlebot-Image Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders User-agent: AdsBot-Google Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders User-agent: AdsBot-Google-Mobile Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Bing / Microsoft Ads User-agent: bingbot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders User-agent: adidxbot Allow: / Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /account Disallow: /orders # Note on TikTok: Bytespider (its general AI scraper) is intentionally # Disallowed in the scraper-block section below. There is no separate # "TikTok Shop crawler" — TikTok pulls product feeds via authenticated # Merchant Center API, not via web crawl. # ──────────────────────────────────────────────────────────────── # Aggressive scraper / SEO-spy bots — disallowed # חסימת בוטי גריפה וריגול תחרותי שמשמשים מעתיקים # ──────────────────────────────────────────────────────────────── User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: AhrefsBot Disallow: / User-agent: AhrefsSiteAudit Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: SeekportBot Disallow: / User-agent: PetalBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: SEOkicks Disallow: / User-agent: Sogou Disallow: / User-agent: serpstatbot Disallow: / User-agent: BacklinkCrawler Disallow: / User-agent: spbot Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: Bytespider Disallow: / User-agent: ImagesiftBot Disallow: /