{
  "schemaVersion": "2.74",
  "lastModified": "2026-10-04",
  "canonical": "https://yangtsunghann.com/data/crawler-policy.json",
  "purpose": "區分搜尋索引、AI 搜尋引用、機器可讀知識、營運技術檔與供應商特定模型訓練控制。V2.56 明確加入 Claude-SearchBot、PerplexityBot 的搜尋存取，並連接外部擷取／crawler 驗收基準。",
  "principles": [
    "公開作品與藝術家資訊以可發現、可引用為優先。",
    "機器可讀知識檔可抓取但維持 noindex，避免成為一般搜尋結果頁。",
    "部署、分析、Consent 等營運技術檔維持不需要 crawler 抓取。",
    "搜尋／引用與模型訓練是不同用途，依供應商控制分開記錄。",
    "robots 政策本身不等於 CDN／WAF 已放行；外部擷取與伺服器紀錄是另一層驗證。"
  ],
  "discovery": {
    "explicitlyAllowedRobots": [
      "Googlebot",
      "Googlebot-Image",
      "OAI-SearchBot",
      "Claude-SearchBot",
      "PerplexityBot"
    ],
    "publicIndexableUrls": 84,
    "sitemap": "https://yangtsunghann.com/sitemap.xml",
    "imageSitemap": "https://yangtsunghann.com/sitemap-artwork.xml",
    "policy": "allow public search/citation discovery"
  },
  "machineReadableKnowledge": {
    "crawlable": true,
    "searchIndexDirective": "noindex, follow",
    "paths": [
      "/artwork/catalog.json",
      "/data/answer-coverage.json",
      "/data/artist-knowledge.json",
      "/data/citation-graph.json",
      "/data/claim-provenance.json",
      "/data/conflict-arbitration.json",
      "/data/crawler-policy.json",
      "/data/data-governance.json",
      "/data/dependency-propagation.json",
      "/data/entity-disambiguation.json",
      "/data/entity-evidence-graph.json",
      "/data/entity-graph.json",
      "/data/evidence-map.json",
      "/data/evidence-quality.json",
      "/data/external-retrieval-benchmark.json",
      "/data/image-graph.json",
      "/data/knowledge-consistency.json",
      "/data/query-intent-map.json",
      "/data/regression-certification.json",
      "/data/revalidation-dry-run.json",
      "/data/revalidation-ledger.json",
      "/data/search-ai-launch-hardening.json",
      "/data/search-identity.json",
      "/data/temporal-truth.json",
      "/llms-full.txt",
      "/llms.txt"
    ]
  },
  "operationalResources": {
    "crawlPolicy": "disallow",
    "searchIndexDirective": "noindex, nofollow",
    "paths": [
      "/deploy-manifest.json",
      "/version.json",
      "/docs/",
      "/data/analytics.json",
      "/data/consent.json",
      "/data/search-console.json"
    ]
  },
  "modelTraining": {
    "policy": "provider_specific",
    "rightsPolicyRegistry": "/data/ai-rights-policy.json",
    "openAI": {
      "searchCrawler": "OAI-SearchBot",
      "searchPolicy": "explicit_allow",
      "trainingCrawler": "GPTBot",
      "trainingPolicy": "explicit_disallow",
      "separatedControls": true
    },
    "google": {
      "searchCrawler": "Googlebot",
      "searchPolicy": "explicit_allow",
      "controlToken": "Google-Extended",
      "effectiveAccess": "inherited_public_default",
      "couplesTrainingAndGrounding": true,
      "policyDecision": "intentional_public_default"
    },
    "anthropic": {
      "searchCrawler": "Claude-SearchBot",
      "searchPolicy": "explicit_allow",
      "trainingCrawler": "ClaudeBot",
      "trainingPolicy": "inherited_public_default",
      "policyDecision": "open_rights_policy_decision"
    },
    "perplexity": {
      "searchCrawler": "PerplexityBot",
      "searchPolicy": "explicit_allow",
      "foundationModelTrainingUse": "provider_docs_state_not_used_for_ai_foundation_model_training"
    }
  },
  "userTriggeredFetchers": {
    "policy": "provider_specific_user_initiated_retrieval",
    "agents": [
      "ChatGPT-User",
      "Claude-User",
      "Perplexity-User"
    ],
    "note": "使用者觸發擷取與自動搜尋 crawler 不同；各供應商對 robots.txt 的適用方式可能不同。"
  },
  "sourceDocs": [
    {
      "provider": "OpenAI",
      "url": "https://developers.openai.com/api/docs/bots",
      "verified": "2026-10-04",
      "note": "OAI-SearchBot is for ChatGPT search; GPTBot is a separate foundation-model training crawler. Search visibility requires crawler access and may also depend on CDN/WAF acceptance."
    },
    {
      "provider": "Google",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verified": "2026-10-04",
      "note": "Googlebot controls Google Search crawling. Google-Extended is a separate product token; it does not control Search inclusion and covers Gemini model-training use plus Gemini/Vertex AI grounding."
    },
    {
      "provider": "Anthropic",
      "url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "verified": "2026-10-04",
      "note": "Anthropic distinguishes ClaudeBot, Claude-SearchBot, and Claude-User. Claude-SearchBot is used for search-result quality; Claude-User is user-initiated retrieval."
    },
    {
      "provider": "Perplexity",
      "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "verified": "2026-10-04",
      "note": "PerplexityBot surfaces and links websites in Perplexity search results and is not used to crawl content for AI foundation models. Perplexity-User is user-initiated retrieval and generally ignores robots.txt."
    }
  ],
  "governanceRegistry": "/data/data-governance.json",
  "visibleContentChanged": false,
  "rightsPolicyRegistry": "/data/ai-rights-policy.json",
  "searchIdentityRegistry": "/data/search-identity.json",
  "externalRetrievalBenchmark": "/data/external-retrieval-benchmark.json",
  "searchAiObservability": "/data/search-ai-observability.json",
  "launchVerification": {
    "verifiedOn": "2026-10-04",
    "targetPublicLaunchDate": "2026-10-10",
    "robotsDirectiveChangesInThisRelease": false,
    "policyChoicesChanged": false,
    "openAISearchBotAllowed": true,
    "openAIGPTBotDisallowed": true,
    "googlebotAllowed": true,
    "googleExtendedPolicy": "inherited_public_default",
    "claudeSearchBotAllowed": true,
    "claudeBotPolicy": "inherited_public_default",
    "perplexityBotAllowed": true,
    "cdnWafVerifiedForNamedBots": false,
    "note": "robots.txt policy was rechecked against current official provider documentation. CDN/WAF acceptance is an external infrastructure layer and is not claimed verified."
  }
}