Line data    Source code

       1              : // Copyright 2026 The Authors
       2              : //
       3              : // Licensed under the Apache License, Version 2.0 (the "License");
       4              : // you may not use this file except in compliance with the License.
       5              : // You may obtain a copy of the License at
       6              : //
       7              : //     https://www.apache.org/licenses/LICENSE-2.0
       8              : //
       9              : // Unless required by applicable law or agreed to in writing, software
      10              : // distributed under the License is distributed on an "AS IS" BASIS,
      11              : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12              : // See the License for the specific language governing permissions and
      13              : // limitations under the License.
      14              : 
      15              : /// Abstract interface for text segmentation.
      16              : ///
      17              : /// Implementations segment a string into word-like tokens, discarding
      18              : /// whitespace and punctuation boundaries. The pipeline then normalises and
      19              : /// stems the returned tokens; this interface is responsible only for the
      20              : /// segmentation step.
      21              : ///
      22              : /// ## Implementations
      23              : ///
      24              : /// This package provides two implementations:
      25              : ///
      26              : /// - [RegExpTokenizer] — pure Dart, suitable for English-language prose and
      27              : ///   common technical identifiers. Zero FFI dependencies.
      28              : /// - [IcuTokenizer] — backed by the system ICU library via FFI. Conforms to
      29              : ///   UAX #29 Unicode Text Segmentation and handles non-Latin scripts (CJK,
      30              : ///   Thai, Arabic, etc.) correctly. Prefer this implementation for
      31              : ///   multi-language use cases.
      32              : ///
      33              : /// The interface is intentionally narrow so the implementation can be swapped
      34              : /// without touching the calling pipeline.
      35              : ///
      36              : /// ## Unicode Text Segmentation
      37              : ///
      38              : /// Conformant implementations should follow UAX #29 Unicode Text Segmentation
      39              : /// rules for word boundaries. [IcuTokenizer] provides a full UAX #29
      40              : /// implementation via the system ICU library.
      41              : ///
      42              : /// ## Example
      43              : ///
      44              : /// ```dart
      45              : /// final tokenizer = RegExpTokenizer();
      46              : /// final tokens = tokenizer.tokenise('Dr. Jekyll and Mr. Hyde');
      47              : /// // → ['Dr', 'Jekyll', 'and', 'Mr', 'Hyde']
      48              : /// ```
      49              : abstract interface class Tokenizer {
      50              :   /// Segments [text] into word tokens.
      51              :   ///
      52              :   /// Returns only word-like spans (letters, numbers, mixed-case identifiers).
      53              :   /// Punctuation, whitespace, and other non-word spans are discarded.
      54              :   ///
      55              :   /// An empty [text] must return an empty list without error.
      56              :   List<String> tokenise(String text);
      57              : }
      58              : 
      59              : /// A tokenised word span together with its character offsets in the
      60              : /// original text.
      61              : ///
      62              : /// [start] is inclusive and [end] is exclusive, both measured in UTF-16 code
      63              : /// units — i.e. Dart `String` index space, matching [String.substring] (so
      64              : /// `text.substring(span.start, span.end) == span.text` always holds).
      65              : final class TokenSpan {
      66              :   /// Creates a [TokenSpan].
      67            2 :   const TokenSpan(this.text, this.start, this.end);
      68              : 
      69              :   /// The token's text — identical to what [Tokenizer.tokenise] would have
      70              :   /// returned for this span.
      71              :   final String text;
      72              : 
      73              :   /// The inclusive start offset of this token in the source text.
      74              :   final int start;
      75              : 
      76              :   /// The exclusive end offset of this token in the source text.
      77              :   final int end;
      78              : 
      79            1 :   @override
      80            4 :   String toString() => 'TokenSpan($text, $start, $end)';
      81              : 
      82            1 :   @override
      83              :   bool operator ==(Object other) =>
      84            1 :       other is TokenSpan &&
      85            3 :       other.text == text &&
      86            3 :       other.start == start &&
      87            3 :       other.end == end;
      88              : 
      89            1 :   @override
      90            4 :   int get hashCode => Object.hash(text, start, end);
      91              : }
      92              : 
      93              : /// A [Tokenizer] that can also report each token's position in the source
      94              : /// text.
      95              : ///
      96              : /// Implemented by [IcuTokenizer] and [RegExpTokenizer] — position data is a
      97              : /// natural byproduct of both algorithms' underlying implementation (ICU's
      98              : /// break iterator and [RegExp] matches both already carry span boundaries).
      99              : /// Not implemented by [BrowserTokenizer]: `Intl.Segmenter`'s JS result does
     100              : /// carry a comparable `index` field, but no consumer needs offsets from the
     101              : /// web tokenizer today — left as a documented future extension rather than
     102              : /// implemented speculatively.
     103              : abstract interface class OffsetTokenizer implements Tokenizer {
     104              :   /// Segments [text] into word tokens with their character offsets.
     105              :   ///
     106              :   /// Equivalent to [tokenise] but additionally reporting each token's
     107              :   /// `(start, end)` span. An empty [text] must return an empty list without
     108              :   /// error.
     109              :   List<TokenSpan> tokeniseSpans(String text);
     110              : }