# ---------------------------------------------------------
# japaneseknifecompany.com – robots.txt
# Updated: 2025-07-16
#
# • One unified ruleset for ALL bots—search engines and AI.
# • Normal pages and assets are crawlable.
# • Sensitive admin / logs / duplicate pages stay hidden.
# ---------------------------------------------------------

###############
# 1. Crawlers covered by these rules
###############
User-agent: *                       # generic
User-agent: Googlebot               # Google Search
User-agent: Bingbot                 # Microsoft Bing
User-agent: Applebot                # Apple Search
User-agent: Yandex                  # Yandex
User-agent: DuckDuckBot             # DuckDuckGo
Allow: /

# --- AI / LLM crawlers (allowed, but follow same exclusions) ---
User-agent: GPTBot                  # OpenAI
User-agent: ChatGPT-User            # OpenAI browser extension
User-agent: Google-Extended         # Google generative-AI feed
User-agent: Anthropic-ai            # Anthropic / Claude
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: pplxbot                 # Perplexity short form
User-agent: CCBot                   # Common Crawl
User-agent: Bytespider              # ByteDance / TikTok
User-agent: Amazonbot               # Amazon AI
User-agent: Applebot-Extended       # Apple generative-AI feed
User-agent: Diffbot
User-agent: YouBot                  # You.com
User-agent: Cohere-ai
User-agent: deepseek
User-agent: OAI-SearchBot
User-agent: OAI-AdsBot
Allow: /

###############
# 2. Directives for everyone above
###############

# WordPress core
Disallow: /wp-admin/
Allow:    /wp-admin/admin-ajax.php

# WordPress REST API (no SEO value)
Disallow: /wp-json/
Disallow: /?rest_route=

# WooCommerce – keep logs / transient / upload buckets private
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/

# WPForms uploads (may contain personal data)
Disallow: /wp-content/uploads/wpforms/

# E-commerce utility pages (thin/duplicate content)
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# Stop endless filter / add-to-cart parameter variants
Disallow: /*add-to-cart=*
Disallow: /*orderby=*
Disallow: /*filter_*
Disallow: /?filter*
Disallow: /?orderby*

# Explicitly allow public product images / assets
Allow: /wp-content/uploads/

###############
# 3. XML sitemap
###############
Sitemap: https://japaneseknifecompany.com/sitemap.xml

# ---------------------------------------------------------
# End of file
# ---------------------------------------------------------