Line data    Source code

       1              : // Copyright 2026 The Authors
       2              : //
       3              : // Licensed under the Apache License, Version 2.0 (the "License");
       4              : // you may not use this file except in compliance with the License.
       5              : // You may obtain a copy of the License at
       6              : //
       7              : //     https://www.apache.org/licenses/LICENSE-2.0
       8              : //
       9              : // Unless required by applicable law or agreed to in writing, software
      10              : // distributed under the License is distributed on an "AS IS" BASIS,
      11              : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12              : // See the License for the specific language governing permissions and
      13              : // limitations under the License.
      14              : 
      15              : import 'tokenizer.dart';
      16              : 
      17              : /// The pure-Dart [Tokenizer] implementation, using [RegExp] for word boundary
      18              : /// detection.
      19              : ///
      20              : /// ## Scope — English and Latin scripts
      21              : ///
      22              : /// This implementation is sufficient for English-language prose and common
      23              : /// technical identifiers (`mTLS`, `0x8004210B`, etc.). It is **not** suitable
      24              : /// for non-Latin scripts (CJK, Thai, Arabic, etc.) where word boundaries do
      25              : /// not follow whitespace rules. For those use cases, prefer [IcuTokenizer],
      26              : /// which uses the system ICU library and conforms to UAX #29 Unicode Text
      27              : /// Segmentation. The [Tokenizer] interface makes that swap transparent to the
      28              : /// calling pipeline.
      29              : ///
      30              : /// ## Example
      31              : ///
      32              : /// ```dart
      33              : /// final tokenizer = RegExpTokenizer();
      34              : /// print(tokenizer.tokenise('Hello, world!')); // ['Hello', 'world']
      35              : /// print(tokenizer.tokenise('')); // []
      36              : /// print(tokenizer.tokenise('mTLS handshake')); // ['mTLS', 'handshake']
      37              : /// ```
      38              : class RegExpTokenizer implements OffsetTokenizer {
      39              :   /// Creates a new [RegExpTokenizer].
      40            3 :   const RegExpTokenizer();
      41              : 
      42              :   /// Matches sequences of Unicode word characters (letters, digits, and
      43              :   /// underscores), optionally allowing internal hyphens and apostrophes so
      44              :   /// that contractions and hyphenated terms are kept intact.
      45              :   ///
      46              :   /// `\w` in Dart's RegExp engine matches `[a-zA-Z0-9_]` only (ASCII).
      47              :   /// For broader Unicode letter support we use `\p{L}` and `\p{N}` with the
      48              :   /// `unicode` flag.
      49            9 :   static final _wordPattern = RegExp(
      50              :     r"[\p{L}\p{N}][\p{L}\p{N}_'\-]*[\p{L}\p{N}]|[\p{L}\p{N}]",
      51              :     unicode: true,
      52              :   );
      53              : 
      54            3 :   @override
      55              :   List<String> tokenise(String text) {
      56            3 :     if (text.isEmpty) return const [];
      57            3 :     return _wordPattern
      58            3 :         .allMatches(text)
      59            9 :         .map((m) => m.group(0)!)
      60            3 :         .toList(growable: false);
      61              :   }
      62              : 
      63            1 :   @override
      64              :   List<TokenSpan> tokeniseSpans(String text) {
      65            1 :     if (text.isEmpty) return const [];
      66            1 :     return _wordPattern
      67            1 :         .allMatches(text)
      68            6 :         .map((m) => TokenSpan(m.group(0)!, m.start, m.end))
      69            1 :         .toList(growable: false);
      70              :   }
      71              : }