{"ok":true,"entity":{"slug":"tokenization","entityType":"concept","name":"トークン化（Tokenization）","canonicalName":"Tokenization","displayName":"トークン化（Tokenization）","category":"AI基礎概念","shortDescription":"テキストをモデルが扱える単位（トークン）へ分割する処理。大規模言語モデルの入力の前処理となる。","alias":["トークナイズ","Tokenization"],"searchKeywords":["tokenization","トークン化","トークン","BPE"],"website":null,"parentEntity":null,"primaryCluster":"ai-company","secondaryClusters":[],"id":"tokenization","verificationStatus":"draft","updatedAt":"2026-07-10T01:03:56.918Z"},"referenceIndex":["P-01-001","P-02-001","P-04-001"]}