{
  "schemaVersion": "2.74",
  "lastModified": "2026-10-04",
  "canonical": "https://yangtsunghann.com/en/data/crawler-policy.json",
  "purpose": "Separates search indexing, AI-search citation, machine-readable knowledge, operational resources, and provider-specific training controls. V2.56 adds explicit Claude-SearchBot and PerplexityBot search access and an external retrieval acceptance benchmark.",
  "principles": [
    "Public artwork and artist information prioritize discovery and citation.",
    "Machine-readable knowledge may be crawled while remaining noindex for normal search-result pages.",
    "Operational deployment/analytics/consent resources remain excluded from crawler access.",
    "Search/citation and model training are separate uses; provider controls are recorded independently.",
    "Crawler policy alone does not prove CDN/WAF acceptance; external retrieval and server logs are separate verification layers."
  ],
  "discovery": {
    "explicitlyAllowedRobots": [
      "Googlebot",
      "Googlebot-Image",
      "OAI-SearchBot",
      "Claude-SearchBot",
      "PerplexityBot"
    ],
    "publicIndexableUrls": 84,
    "sitemap": "https://yangtsunghann.com/sitemap.xml",
    "imageSitemap": "https://yangtsunghann.com/sitemap-artwork.xml",
    "policy": "allow public search/citation discovery"
  },
  "machineReadableKnowledge": {
    "crawlable": true,
    "searchIndexDirective": "noindex, follow",
    "paths": [
      "/artwork/catalog.json",
      "/data/answer-coverage.json",
      "/data/artist-knowledge.json",
      "/data/citation-graph.json",
      "/data/claim-provenance.json",
      "/data/conflict-arbitration.json",
      "/data/crawler-policy.json",
      "/data/data-governance.json",
      "/data/dependency-propagation.json",
      "/data/entity-disambiguation.json",
      "/data/entity-evidence-graph.json",
      "/data/entity-graph.json",
      "/data/evidence-map.json",
      "/data/evidence-quality.json",
      "/data/external-retrieval-benchmark.json",
      "/data/image-graph.json",
      "/data/knowledge-consistency.json",
      "/data/query-intent-map.json",
      "/data/regression-certification.json",
      "/data/revalidation-dry-run.json",
      "/data/revalidation-ledger.json",
      "/data/temporal-truth.json",
      "/en/data/search-ai-launch-hardening.json",
      "/en/data/search-identity.json",
      "/llms-full.txt",
      "/llms.txt"
    ]
  },
  "operationalResources": {
    "crawlPolicy": "disallow",
    "searchIndexDirective": "noindex, nofollow",
    "paths": [
      "/deploy-manifest.json",
      "/version.json",
      "/docs/",
      "/data/analytics.json",
      "/data/consent.json",
      "/data/search-console.json"
    ]
  },
  "modelTraining": {
    "policy": "provider_specific",
    "rightsPolicyRegistry": "/en/data/ai-rights-policy.json",
    "openAI": {
      "searchCrawler": "OAI-SearchBot",
      "searchPolicy": "explicit_allow",
      "trainingCrawler": "GPTBot",
      "trainingPolicy": "explicit_disallow",
      "separatedControls": true
    },
    "google": {
      "searchCrawler": "Googlebot",
      "searchPolicy": "explicit_allow",
      "controlToken": "Google-Extended",
      "effectiveAccess": "inherited_public_default",
      "couplesTrainingAndGrounding": true,
      "policyDecision": "intentional_public_default"
    },
    "anthropic": {
      "searchCrawler": "Claude-SearchBot",
      "searchPolicy": "explicit_allow",
      "trainingCrawler": "ClaudeBot",
      "trainingPolicy": "inherited_public_default",
      "policyDecision": "open_rights_policy_decision"
    },
    "perplexity": {
      "searchCrawler": "PerplexityBot",
      "searchPolicy": "explicit_allow",
      "foundationModelTrainingUse": "provider_docs_state_not_used_for_ai_foundation_model_training"
    }
  },
  "userTriggeredFetchers": {
    "policy": "provider_specific_user_initiated_retrieval",
    "agents": [
      "ChatGPT-User",
      "Claude-User",
      "Perplexity-User"
    ],
    "note": "User-initiated fetchers are distinct from automatic search crawlers; provider-specific robots behavior may differ."
  },
  "sourceDocs": [
    {
      "provider": "OpenAI",
      "url": "https://developers.openai.com/api/docs/bots",
      "verified": "2026-10-04",
      "note": "OAI-SearchBot is for ChatGPT search; GPTBot is a separate foundation-model training crawler. Search visibility requires crawler access and may also depend on CDN/WAF acceptance."
    },
    {
      "provider": "Google",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verified": "2026-10-04",
      "note": "Googlebot controls Google Search crawling. Google-Extended is a separate product token; it does not control Search inclusion and covers Gemini model-training use plus Gemini/Vertex AI grounding."
    },
    {
      "provider": "Anthropic",
      "url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "verified": "2026-10-04",
      "note": "Anthropic distinguishes ClaudeBot, Claude-SearchBot, and Claude-User. Claude-SearchBot is used for search-result quality; Claude-User is user-initiated retrieval."
    },
    {
      "provider": "Perplexity",
      "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "verified": "2026-10-04",
      "note": "PerplexityBot surfaces and links websites in Perplexity search results and is not used to crawl content for AI foundation models. Perplexity-User is user-initiated retrieval and generally ignores robots.txt."
    }
  ],
  "governanceRegistry": "/data/data-governance.json",
  "visibleContentChanged": false,
  "authoritativeRegistry": "https://yangtsunghann.com/data/crawler-policy.json",
  "role": "localized derivative",
  "rightsPolicyRegistry": "/en/data/ai-rights-policy.json",
  "searchIdentityRegistry": "/en/data/search-identity.json",
  "externalRetrievalBenchmark": "/en/data/external-retrieval-benchmark.json",
  "searchAiObservability": "/en/data/search-ai-observability.json",
  "launchVerification": {
    "verifiedOn": "2026-10-04",
    "targetPublicLaunchDate": "2026-10-10",
    "robotsDirectiveChangesInThisRelease": false,
    "policyChoicesChanged": false,
    "openAISearchBotAllowed": true,
    "openAIGPTBotDisallowed": true,
    "googlebotAllowed": true,
    "googleExtendedPolicy": "inherited_public_default",
    "claudeSearchBotAllowed": true,
    "claudeBotPolicy": "inherited_public_default",
    "perplexityBotAllowed": true,
    "cdnWafVerifiedForNamedBots": false,
    "note": "robots.txt policy was rechecked against current official provider documentation. CDN/WAF acceptance is an external infrastructure layer and is not claimed verified."
  }
}