Line data Source code
1 : // Copyright 2026 The Authors
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // https://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : import 'dart:ffi';
16 : import 'dart:io';
17 :
18 : import 'package:ffi/ffi.dart';
19 :
20 : import 'tokenizer.dart';
21 :
22 : // ---------------------------------------------------------------------------
23 : // ICU constants
24 : // ---------------------------------------------------------------------------
25 :
26 : /// UBreakIteratorType value for word boundary analysis.
27 : const int _ubrkWord = 2;
28 :
29 : /// Sentinel returned by ubrk_next() when iteration is complete.
30 : ///
31 : /// ICU defines UBRK_DONE as (int32_t)0xFFFFFFFF — i.e. -1 in signed form.
32 : const int _ubrkDone = -1;
33 :
34 : // ---------------------------------------------------------------------------
35 : // Native function typedefs
36 : // ---------------------------------------------------------------------------
37 :
38 : /// `ubrk_open` — allocates a UBreakIterator.
39 : ///
40 : /// Passing address 0 for [locale] selects ICU's default (root) locale, which
41 : /// is sufficient for script-level word boundary rules.
42 : typedef _UbrkOpenNative = Pointer<Void> Function(
43 : Int32 type,
44 : Pointer<Utf8> locale,
45 : Pointer<Uint16> text,
46 : Int32 textLength,
47 : Pointer<Int32> status,
48 : );
49 : typedef _UbrkOpen = Pointer<Void> Function(
50 : int type,
51 : Pointer<Utf8> locale,
52 : Pointer<Uint16> text,
53 : int textLength,
54 : Pointer<Int32> status,
55 : );
56 :
57 : /// `ubrk_next` — advance to the next boundary; returns position or [_ubrkDone].
58 : typedef _UbrkNextNative = Int32 Function(Pointer<Void> bi);
59 : typedef _UbrkNext = int Function(Pointer<Void> bi);
60 :
61 : /// `ubrk_close` — release the UBreakIterator.
62 : typedef _UbrkCloseNative = Void Function(Pointer<Void> bi);
63 : typedef _UbrkClose = void Function(Pointer<Void> bi);
64 :
65 : // ---------------------------------------------------------------------------
66 : // Library loader
67 : // ---------------------------------------------------------------------------
68 :
69 : /// Opens the system ICU library appropriate for the current platform.
70 : ///
71 : /// ICU is bundled with every target OS supported by this package:
72 : ///
73 : /// | Platform | Library |
74 : /// |-------------|--------------------------------------|
75 : /// | macOS / iOS | libicucore.dylib (ships with OS) |
76 : /// | Android | libicuuc.so (NDK) |
77 : /// | Linux | libicuuc.so.NN (widely packaged) |
78 : /// | Windows | icu.dll (Windows 10+) |
79 : ///
80 : /// [platform] defaults to [Platform.operatingSystem]. Pass an explicit value
81 : /// to exercise non-native library-loading paths in tests.
82 : ///
83 : /// Throws [UnsupportedError] if no matching library can be found.
84 2 : DynamicLibrary _openIcuLibrary([String? platform]) {
85 2 : platform ??= Platform.operatingSystem;
86 :
87 4 : if (platform == 'macos' || platform == 'ios') {
88 1 : return DynamicLibrary.open('libicucore.dylib');
89 : }
90 :
91 2 : if (platform == 'android') {
92 0 : return DynamicLibrary.open('libicuuc.so');
93 : }
94 :
95 2 : if (platform == 'linux') {
96 : // ubrk_open and other break-iterator symbols live in libicuuc (Common),
97 : // not libicui18n. The unversioned symlink requires the -dev package;
98 : // fall back through versioned names common across distributions.
99 : const candidates = [
100 : 'libicuuc.so',
101 : 'libicuuc.so.76',
102 : 'libicuuc.so.74',
103 : 'libicuuc.so.73',
104 : 'libicuuc.so.72',
105 : 'libicuuc.so.70',
106 : 'libicuuc.so.67',
107 : 'libicuuc.so.66',
108 : ];
109 4 : for (final name in candidates) {
110 : try {
111 2 : return DynamicLibrary.open(name);
112 : } catch (_) {
113 : // try next candidate
114 : }
115 : }
116 0 : throw UnsupportedError(
117 : 'Could not find libicuuc on this Linux system. '
118 : 'Install libicu-dev (Debian/Ubuntu) or icu (Arch/Fedora).',
119 : );
120 : }
121 :
122 1 : if (platform == 'windows') {
123 : const candidates = ['icu.dll', 'icuuc.dll'];
124 2 : for (final name in candidates) {
125 : try {
126 1 : return DynamicLibrary.open(name);
127 : } catch (_) {
128 : // try next candidate
129 : }
130 : }
131 1 : throw UnsupportedError('Could not find ICU DLL on this Windows system.');
132 : }
133 :
134 2 : throw UnsupportedError('IcuTokenizer is not supported on $platform.');
135 : }
136 :
137 : // ---------------------------------------------------------------------------
138 : // ICU symbol-suffix resolver
139 : // ---------------------------------------------------------------------------
140 :
141 : /// Returns the version suffix appended to ICU symbols on this system, or `''`.
142 : ///
143 : /// Some distributions (older Debian/Ubuntu) disable ICU symbol renaming so
144 : /// `ubrk_open` is exported as-is. Others (Fedora, Debian Trixie+) use ICU's
145 : /// default renaming, which appends the major version number (e.g. `_76`).
146 2 : String _icuSymbolSuffix(DynamicLibrary lib) {
147 : const versionsToTry = [0, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64];
148 4 : for (final v in versionsToTry) {
149 4 : final suffix = v == 0 ? '' : '_$v';
150 : try {
151 4 : lib.lookup<NativeFunction<_UbrkOpenNative>>('ubrk_open$suffix');
152 : return suffix;
153 : } catch (_) {
154 : // try next
155 : }
156 : }
157 0 : throw UnsupportedError(
158 : 'Could not find ubrk_open[_NN] in the loaded ICU library. '
159 : 'The library may be incomplete or use an unsupported symbol renaming scheme.',
160 : );
161 : }
162 :
163 : // ---------------------------------------------------------------------------
164 : // IcuTokenizer
165 : // ---------------------------------------------------------------------------
166 :
167 : /// An [OffsetTokenizer] backed by the ICU C library's UBRK_WORD break
168 : /// iterator.
169 : ///
170 : /// Conforms to UAX #29 Unicode Text Segmentation and handles non-Latin scripts
171 : /// (CJK, Thai, Arabic, etc.) correctly. This is the preferred implementation
172 : /// for multi-language use cases.
173 : ///
174 : /// ## Deployment
175 : ///
176 : /// ICU is a system library on all of this package's target platforms — no
177 : /// bundling is required and there is no App Store risk:
178 : ///
179 : /// | Platform | Library |
180 : /// |-------------|--------------------------------------|
181 : /// | macOS / iOS | libicucore.dylib (ships with OS) |
182 : /// | Android | libicuuc.so (NDK) |
183 : /// | Linux | libicuuc.so.NN (widely packaged) |
184 : /// | Windows | icu.dll (Windows 10+) |
185 : ///
186 : /// ## Platform note — ubrk_getRuleStatus
187 : ///
188 : /// Apple's libicucore does not include UAX #29 rule-status tags in its
189 : /// compiled word break rules, so `ubrk_getRuleStatus()` returns non-standard
190 : /// values on macOS/iOS. This implementation therefore uses Dart's own Unicode
191 : /// `RegExp` for span classification rather than relying on rule-status codes.
192 : /// Boundary *positions* from the ICU iterator are correct on all platforms.
193 : ///
194 : /// Construct once and reuse — the FFI bindings are resolved at construction
195 : /// time. Each call to [tokeniseSpans] (which [tokenise] delegates to)
196 : /// allocates a temporary native UTF-16 buffer and releases it before
197 : /// returning.
198 : ///
199 : /// Throws [UnsupportedError] if the ICU library cannot be found or if the
200 : /// required symbols are absent.
201 : class IcuTokenizer implements OffsetTokenizer {
202 : // Retain the DynamicLibrary reference to prevent the OS from unloading the
203 : // library while this tokenizer is alive.
204 : // ignore: unused_field
205 : final DynamicLibrary _lib;
206 : final _UbrkOpen _ubrkOpen;
207 : final _UbrkNext _ubrkNext;
208 : final _UbrkClose _ubrkClose;
209 :
210 : /// Opens the system ICU library and resolves the FFI symbols.
211 : ///
212 : /// Throws [UnsupportedError] if the library cannot be found on this platform.
213 6 : factory IcuTokenizer() => IcuTokenizer._fromLib(_openIcuLibrary());
214 :
215 : /// Creates an [IcuTokenizer] that loads the ICU library for [platform].
216 : ///
217 : /// [platform] must be a [Platform.operatingSystem] string such as `'linux'`,
218 : /// `'android'`, or `'windows'`. This constructor lets tests exercise each
219 : /// library-loading branch on a development machine without requiring the
220 : /// native platform.
221 1 : factory IcuTokenizer.forPlatform(String platform) =>
222 1 : IcuTokenizer._fromLib(_openIcuLibrary(platform));
223 :
224 2 : factory IcuTokenizer._fromLib(DynamicLibrary lib) {
225 2 : final s = _icuSymbolSuffix(lib);
226 2 : return IcuTokenizer._(
227 : lib,
228 2 : lib.lookupFunction<_UbrkOpenNative, _UbrkOpen>('ubrk_open$s'),
229 2 : lib.lookupFunction<_UbrkNextNative, _UbrkNext>('ubrk_next$s'),
230 2 : lib.lookupFunction<_UbrkCloseNative, _UbrkClose>('ubrk_close$s'),
231 : );
232 : }
233 :
234 2 : IcuTokenizer._(this._lib, this._ubrkOpen, this._ubrkNext, this._ubrkClose);
235 :
236 : // Matches any Unicode letter or digit — used to classify ICU spans.
237 : //
238 : // NOTE: ubrk_getRuleStatus() is NOT used for span classification. Apple's
239 : // libicucore does not include the UAX #29 rule-status tags in its compiled
240 : // word break rules, so the function returns 0 for all letter/number spans
241 : // and non-zero for certain whitespace sequences — the inverse of the
242 : // upstream ICU convention. Character-based classification is both more
243 : // portable and simpler.
244 6 : static final _hasWordChar = RegExp(r'[\p{L}\p{N}]', unicode: true);
245 :
246 : // Strips leading/trailing non-letter/non-digit characters from a span.
247 : // Some ICU builds (including Apple's) group adjacent punctuation into the
248 : // same span as the word (e.g. "Hello," rather than "Hello" + ",").
249 6 : static final _leadingNonWord = RegExp(r'^[^\p{L}\p{N}]+', unicode: true);
250 6 : static final _trailingNonWord = RegExp(r'[^\p{L}\p{N}]+$', unicode: true);
251 :
252 2 : @override
253 : List<String> tokenise(String text) =>
254 10 : tokeniseSpans(text).map((s) => s.text).toList(growable: false);
255 :
256 2 : @override
257 : List<TokenSpan> tokeniseSpans(String text) {
258 2 : if (text.isEmpty) return const [];
259 :
260 2 : final codeUnits = text.codeUnits;
261 2 : final len = codeUnits.length;
262 :
263 : // Allocate a native UTF-16 buffer and an error-code cell.
264 : final textBuf = calloc<Uint16>(len);
265 : final statusBuf = calloc<Int32>();
266 :
267 : try {
268 : // Copy the Dart string's UTF-16 code units into native memory.
269 : // Dart strings are UTF-16 encoded; codeUnits gives the correct uint16_t
270 : // values for all characters, including supplementary characters encoded
271 : // as surrogate pairs.
272 4 : for (var i = 0; i < len; i++) {
273 4 : textBuf[i] = codeUnits[i];
274 : }
275 :
276 : // Open a UBRK_WORD iterator. Passing address 0 for locale means
277 : // "default locale"; for script-level word breaking this is fine.
278 4 : final bi = _ubrkOpen(
279 : _ubrkWord,
280 2 : Pointer<Utf8>.fromAddress(0), // nullptr → default locale
281 : textBuf,
282 : len,
283 : statusBuf,
284 : );
285 :
286 4 : _checkStatus(statusBuf.value, 'ubrk_open');
287 :
288 : try {
289 2 : final spans = <TokenSpan>[];
290 : var start = 0;
291 :
292 : while (true) {
293 4 : final end = _ubrkNext(bi);
294 2 : if (end == _ubrkDone) break;
295 :
296 2 : final span = text.substring(start, end);
297 :
298 : // Include the span only if it contains at least one letter or digit.
299 : // Then strip any punctuation that was grouped at either end,
300 : // tracking exactly how many characters were trimmed from each side
301 : // so the reported offsets describe the trimmed word, not the raw
302 : // ICU span. `_leadingNonWord`/`_trailingNonWord` are anchored
303 : // (`^`/`$`), so `firstMatch` finds precisely what `replaceFirst`
304 : // would have removed.
305 4 : if (_hasWordChar.hasMatch(span)) {
306 6 : final leadingTrim = _leadingNonWord.firstMatch(span)?.end ?? 0;
307 4 : final trailingMatch = _trailingNonWord.firstMatch(span);
308 : final trailingTrim = trailingMatch == null
309 : ? 0
310 6 : : span.length - trailingMatch.start;
311 2 : final word = span.substring(
312 : leadingTrim,
313 4 : span.length - trailingTrim,
314 : );
315 2 : if (word.isNotEmpty) {
316 2 : spans.add(
317 6 : TokenSpan(word, start + leadingTrim, end - trailingTrim),
318 : );
319 : }
320 : }
321 :
322 : start = end;
323 : }
324 :
325 : return spans;
326 : } finally {
327 4 : _ubrkClose(bi);
328 : }
329 : } finally {
330 2 : calloc.free(textBuf);
331 2 : calloc.free(statusBuf);
332 : }
333 : }
334 :
335 : /// Throws [StateError] if [statusCode] indicates a fatal ICU error.
336 : ///
337 : /// ICU's UErrorCode convention (unicode/utypes.h):
338 : /// < 0 warnings (non-fatal — e.g. U_USING_DEFAULT_WARNING = -127)
339 : /// = 0 U_ZERO_ERROR (success)
340 : /// > 0 errors (fatal — e.g. U_ILLEGAL_ARGUMENT_ERROR = 1)
341 2 : static void _checkStatus(int statusCode, String fn) {
342 2 : if (statusCode > 0) {
343 0 : throw StateError('ICU error in $fn: UErrorCode $statusCode');
344 : }
345 : }
346 : }
|
