Line data    Source code

       1              : // Copyright 2026 The Authors.
       2              : //
       3              : // Licensed under the Apache License, Version 2.0 (the "License");
       4              : // you may not use this file except in compliance with the License.
       5              : // You may obtain a copy of the License at
       6              : //
       7              : //     https://www.apache.org/licenses/LICENSE-2.0
       8              : //
       9              : // Unless required by applicable law or agreed to in writing, software
      10              : // distributed under the License is distributed on an "AS IS" BASIS,
      11              : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12              : // See the License for the specific language governing permissions and
      13              : // limitations under the License.
      14              : 
      15              : import 'script_ranges.g.dart' show scriptOfRune;
      16              : 
      17              : /// Maximum number of runes examined from the start of the input.
      18              : ///
      19              : /// Bounds cost on very large inputs. This is a defensive cap, not a tuned
      20              : /// limit — text this package is expected to see (document titles, search
      21              : /// queries, vault chunks) is far shorter, so the cap is not expected to bind
      22              : /// in practice. Mirrors the sampling pattern in `betto_charset_detector`.
      23              : const int sampleRuneCap = 5000;
      24              : 
      25              : /// ISO 15924 code for the Unicode `Common` script property value — script-
      26              : /// neutral codepoints (whitespace, digits, punctuation, symbols) that carry
      27              : /// no language signal.
      28              : const String _commonScript = 'Zyyy';
      29              : 
      30              : /// ISO 15924 code for the Unicode `Inherited` script property value —
      31              : /// combining marks that inherit their script from the preceding base
      32              : /// character, also carrying no independent signal.
      33              : const String _inheritedScript = 'Zinh';
      34              : 
      35              : /// ISO 15924 code for the Hiragana script.
      36              : const String hiraganaScript = 'Hira';
      37              : 
      38              : /// ISO 15924 code for the Katakana script.
      39              : const String katakanaScript = 'Kana';
      40              : 
      41              : /// ISO 15924 code for the Han script (shared by Chinese and — together with
      42              : /// Hiragana/Katakana — Japanese).
      43              : const String hanScript = 'Hani';
      44              : 
      45              : /// The result of a single-pass scan of a text's Unicode scripts.
      46              : final class ScriptScan {
      47              :   /// The most common script among the input's non-neutral letter codepoints,
      48              :   /// or `null` if none were found (empty, whitespace-only,
      49              :   /// digits/punctuation/emoji-only input).
      50              :   final String? dominantScript;
      51              : 
      52              :   /// Whether any Hiragana or Katakana codepoint was present.
      53              :   ///
      54              :   /// Tracked independently of [dominantScript]'s tally because Japanese text
      55              :   /// is often Han-majority by raw codepoint count — kana presence, not the
      56              :   /// dominant-script count, is the decisive Japanese/Chinese signal.
      57              :   final bool hasKana;
      58              : 
      59            3 :   const ScriptScan({required this.dominantScript, required this.hasKana});
      60              : }
      61              : 
      62              : /// Scans up to [sampleRuneCap] runes of [text], returning both the dominant
      63              : /// script and whether any kana codepoint was present, from a single pass.
      64              : ///
      65              : /// Computing both signals together avoids scanning `text.runes` twice when a
      66              : /// caller (e.g. `CompositeBackend`) needs both.
      67            3 : ScriptScan scanScript(String text) {
      68            3 :   final tally = <String, int>{};
      69              :   var hasKana = false;
      70              :   var examined = 0;
      71              : 
      72            6 :   for (final rune in text.runes) {
      73            3 :     if (examined >= sampleRuneCap) break;
      74            3 :     examined++;
      75              : 
      76            3 :     final code = scriptOfRune(rune);
      77            6 :     if (code == null || code == _commonScript || code == _inheritedScript) {
      78              :       continue;
      79              :     }
      80            6 :     if (code == hiraganaScript || code == katakanaScript) {
      81              :       hasKana = true;
      82              :     }
      83            9 :     tally[code] = (tally[code] ?? 0) + 1;
      84              :   }
      85              : 
      86            3 :   if (tally.isEmpty) {
      87            3 :     return ScriptScan(dominantScript: null, hasKana: hasKana);
      88              :   }
      89              : 
      90            6 :   var bestCode = tally.keys.first;
      91            3 :   var bestCount = tally[bestCode]!;
      92            6 :   for (final entry in tally.entries) {
      93            6 :     if (entry.value > bestCount) {
      94            2 :       bestCode = entry.key;
      95            2 :       bestCount = entry.value;
      96              :     }
      97              :   }
      98            3 :   return ScriptScan(dominantScript: bestCode, hasKana: hasKana);
      99              : }
     100              : 
     101              : /// Cheap, script-only classification of [text].
     102              : ///
     103              : /// Returns a 4-letter ISO 15924 script code (e.g. `"Latn"`, `"Cyrl"`,
     104              : /// `"Hani"`) for the most common script among the input's letter codepoints,
     105              : /// or `null` if the input has no scripted letters.
     106              : ///
     107              : /// This is a convenience wrapper around [scanScript] for callers that only
     108              : /// need the dominant script, not the kana signal.
     109            6 : String? dominantScript(String text) => scanScript(text).dominantScript;