Line data    Source code

       1              : // Copyright 2026 The Authors
       2              : //
       3              : // Licensed under the Apache License, Version 2.0 (the "License");
       4              : // you may not use this file except in compliance with the License.
       5              : // You may obtain a copy of the License at
       6              : //
       7              : //      https://www.apache.org/licenses/LICENSE-2.0
       8              : //
       9              : // Unless required by applicable law or agreed to in writing, software
      10              : // distributed under the License is distributed on an "AS IS" BASIS,
      11              : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12              : // See the License for the specific language governing permissions and
      13              : // limitations under the License.
      14              : 
      15              : import 'package:betto_icu/betto_icu.dart';
      16              : 
      17              : /// Returns the default [Tokenizer] for native platforms.
      18              : ///
      19              : /// On native, [IcuTokenizer] is used: it delegates to the system ICU library
      20              : /// via FFI and conforms to UAX #29 Unicode Text Segmentation. This handles
      21              : /// non-Latin scripts (CJK, Thai, Arabic, etc.) correctly, making it suitable
      22              : /// for multi-language content. ICU is a system library on every native target:
      23              : /// `libicucore.dylib` on macOS/iOS, `libicuuc.so` on Android/Linux, and
      24              : /// `icu.dll` on Windows — no bundling is required.
      25              : ///
      26              : /// Use [RegExpTokenizer] directly when FFI is unavailable or a pure-Dart,
      27              : /// English-only fallback is explicitly needed.
      28              : ///
      29              : /// The companion file `default_tokenizer_web.dart` is selected instead when
      30              : /// running on web via the conditional export in `lexical.dart`.
      31            2 : Tokenizer createDefaultTokenizer() => IcuTokenizer();