Line data Source code
1 : // Copyright 2026 The Authors
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // https://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : import 'dart:ffi';
16 : import 'dart:io';
17 :
18 : import 'package:ffi/ffi.dart';
19 :
20 : import 'tokenizer.dart';
21 :
22 : // ---------------------------------------------------------------------------
23 : // ICU constants
24 : // ---------------------------------------------------------------------------
25 :
26 : /// UBreakIteratorType value for word boundary analysis.
27 : const int _ubrkWord = 2;
28 :
29 : /// Sentinel returned by ubrk_next() when iteration is complete.
30 : ///
31 : /// ICU defines UBRK_DONE as (int32_t)0xFFFFFFFF — i.e. -1 in signed form.
32 : const int _ubrkDone = -1;
33 :
34 : // ---------------------------------------------------------------------------
35 : // Native function typedefs
36 : // ---------------------------------------------------------------------------
37 :
38 : /// `ubrk_open` — allocates a UBreakIterator.
39 : ///
40 : /// Passing address 0 for [locale] selects ICU's default (root) locale, which
41 : /// is sufficient for script-level word boundary rules.
42 : typedef _UbrkOpenNative =
43 : Pointer<Void> Function(
44 : Int32 type,
45 : Pointer<Utf8> locale,
46 : Pointer<Uint16> text,
47 : Int32 textLength,
48 : Pointer<Int32> status,
49 : );
50 : typedef _UbrkOpen =
51 : Pointer<Void> Function(
52 : int type,
53 : Pointer<Utf8> locale,
54 : Pointer<Uint16> text,
55 : int textLength,
56 : Pointer<Int32> status,
57 : );
58 :
59 : /// `ubrk_next` — advance to the next boundary; returns position or [_ubrkDone].
60 : typedef _UbrkNextNative = Int32 Function(Pointer<Void> bi);
61 : typedef _UbrkNext = int Function(Pointer<Void> bi);
62 :
63 : /// `ubrk_close` — release the UBreakIterator.
64 : typedef _UbrkCloseNative = Void Function(Pointer<Void> bi);
65 : typedef _UbrkClose = void Function(Pointer<Void> bi);
66 :
67 : // ---------------------------------------------------------------------------
68 : // Library loader
69 : // ---------------------------------------------------------------------------
70 :
71 : /// Opens the system ICU library appropriate for the current platform.
72 : ///
73 : /// ICU is bundled with every target OS supported by this package:
74 : ///
75 : /// | Platform | Library |
76 : /// |-------------|--------------------------------------|
77 : /// | macOS / iOS | libicucore.dylib (ships with OS) |
78 : /// | Android | libicuuc.so (NDK) |
79 : /// | Linux | libicuuc.so.NN (widely packaged) |
80 : /// | Windows | icu.dll (Windows 10+) |
81 : ///
82 : /// [platform] defaults to [Platform.operatingSystem]. Pass an explicit value
83 : /// to exercise non-native library-loading paths in tests.
84 : ///
85 : /// Throws [UnsupportedError] if no matching library can be found.
86 2 : DynamicLibrary _openIcuLibrary([String? platform]) {
87 2 : platform ??= Platform.operatingSystem;
88 :
89 4 : if (platform == 'macos' || platform == 'ios') {
90 1 : return DynamicLibrary.open('libicucore.dylib');
91 : }
92 :
93 2 : if (platform == 'android') {
94 0 : return DynamicLibrary.open('libicuuc.so');
95 : }
96 :
97 2 : if (platform == 'linux') {
98 : // ubrk_open and other break-iterator symbols live in libicuuc (Common),
99 : // not libicui18n. The unversioned symlink requires the -dev package;
100 : // fall back through versioned names common across distributions.
101 : const candidates = [
102 : 'libicuuc.so',
103 : 'libicuuc.so.76',
104 : 'libicuuc.so.74',
105 : 'libicuuc.so.73',
106 : 'libicuuc.so.72',
107 : 'libicuuc.so.70',
108 : 'libicuuc.so.67',
109 : 'libicuuc.so.66',
110 : ];
111 4 : for (final name in candidates) {
112 : try {
113 2 : return DynamicLibrary.open(name);
114 : } catch (_) {
115 : // try next candidate
116 : }
117 : }
118 0 : throw UnsupportedError(
119 : 'Could not find libicuuc on this Linux system. '
120 : 'Install libicu-dev (Debian/Ubuntu) or icu (Arch/Fedora).',
121 : );
122 : }
123 :
124 1 : if (platform == 'windows') {
125 : const candidates = ['icu.dll', 'icuuc.dll'];
126 2 : for (final name in candidates) {
127 : try {
128 1 : return DynamicLibrary.open(name);
129 : } catch (_) {
130 : // try next candidate
131 : }
132 : }
133 1 : throw UnsupportedError('Could not find ICU DLL on this Windows system.');
134 : }
135 :
136 2 : throw UnsupportedError('IcuTokenizer is not supported on $platform.');
137 : }
138 :
139 : // ---------------------------------------------------------------------------
140 : // ICU symbol-suffix resolver
141 : // ---------------------------------------------------------------------------
142 :
143 : /// Returns the version suffix appended to ICU symbols on this system, or `''`.
144 : ///
145 : /// Some distributions (older Debian/Ubuntu) disable ICU symbol renaming so
146 : /// `ubrk_open` is exported as-is. Others (Fedora, Debian Trixie+) use ICU's
147 : /// default renaming, which appends the major version number (e.g. `_76`).
148 2 : String _icuSymbolSuffix(DynamicLibrary lib) {
149 : const versionsToTry = [0, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64];
150 4 : for (final v in versionsToTry) {
151 4 : final suffix = v == 0 ? '' : '_$v';
152 : try {
153 4 : lib.lookup<NativeFunction<_UbrkOpenNative>>('ubrk_open$suffix');
154 : return suffix;
155 : } catch (_) {
156 : // try next
157 : }
158 : }
159 0 : throw UnsupportedError(
160 : 'Could not find ubrk_open[_NN] in the loaded ICU library. '
161 : 'The library may be incomplete or use an unsupported symbol renaming scheme.',
162 : );
163 : }
164 :
165 : // ---------------------------------------------------------------------------
166 : // IcuTokenizer
167 : // ---------------------------------------------------------------------------
168 :
169 : /// An [OffsetTokenizer] backed by the ICU C library's UBRK_WORD break
170 : /// iterator.
171 : ///
172 : /// Conforms to UAX #29 Unicode Text Segmentation and handles non-Latin scripts
173 : /// (CJK, Thai, Arabic, etc.) correctly. This is the preferred implementation
174 : /// for multi-language use cases.
175 : ///
176 : /// ## Deployment
177 : ///
178 : /// ICU is a system library on all of this package's target platforms — no
179 : /// bundling is required and there is no App Store risk:
180 : ///
181 : /// | Platform | Library |
182 : /// |-------------|--------------------------------------|
183 : /// | macOS / iOS | libicucore.dylib (ships with OS) |
184 : /// | Android | libicuuc.so (NDK) |
185 : /// | Linux | libicuuc.so.NN (widely packaged) |
186 : /// | Windows | icu.dll (Windows 10+) |
187 : ///
188 : /// ## Platform note — ubrk_getRuleStatus
189 : ///
190 : /// Apple's libicucore does not include UAX #29 rule-status tags in its
191 : /// compiled word break rules, so `ubrk_getRuleStatus()` returns non-standard
192 : /// values on macOS/iOS. This implementation therefore uses Dart's own Unicode
193 : /// `RegExp` for span classification rather than relying on rule-status codes.
194 : /// Boundary *positions* from the ICU iterator are correct on all platforms.
195 : ///
196 : /// Construct once and reuse — the FFI bindings are resolved at construction
197 : /// time. Each call to [tokeniseSpans] (which [tokenise] delegates to)
198 : /// allocates a temporary native UTF-16 buffer and releases it before
199 : /// returning.
200 : ///
201 : /// Throws [UnsupportedError] if the ICU library cannot be found or if the
202 : /// required symbols are absent.
203 : class IcuTokenizer implements OffsetTokenizer {
204 : // Retain the DynamicLibrary reference to prevent the OS from unloading the
205 : // library while this tokenizer is alive.
206 : // ignore: unused_field
207 : final DynamicLibrary _lib;
208 : final _UbrkOpen _ubrkOpen;
209 : final _UbrkNext _ubrkNext;
210 : final _UbrkClose _ubrkClose;
211 :
212 : /// Opens the system ICU library and resolves the FFI symbols.
213 : ///
214 : /// Throws [UnsupportedError] if the library cannot be found on this platform.
215 6 : factory IcuTokenizer() => IcuTokenizer._fromLib(_openIcuLibrary());
216 :
217 : /// Creates an [IcuTokenizer] that loads the ICU library for [platform].
218 : ///
219 : /// [platform] must be a [Platform.operatingSystem] string such as `'linux'`,
220 : /// `'android'`, or `'windows'`. This constructor lets tests exercise each
221 : /// library-loading branch on a development machine without requiring the
222 : /// native platform.
223 1 : factory IcuTokenizer.forPlatform(String platform) =>
224 1 : IcuTokenizer._fromLib(_openIcuLibrary(platform));
225 :
226 2 : factory IcuTokenizer._fromLib(DynamicLibrary lib) {
227 2 : final s = _icuSymbolSuffix(lib);
228 2 : return IcuTokenizer._(
229 : lib,
230 2 : lib.lookupFunction<_UbrkOpenNative, _UbrkOpen>('ubrk_open$s'),
231 2 : lib.lookupFunction<_UbrkNextNative, _UbrkNext>('ubrk_next$s'),
232 2 : lib.lookupFunction<_UbrkCloseNative, _UbrkClose>('ubrk_close$s'),
233 : );
234 : }
235 :
236 2 : IcuTokenizer._(this._lib, this._ubrkOpen, this._ubrkNext, this._ubrkClose);
237 :
238 : // Matches any Unicode letter or digit — used to classify ICU spans.
239 : //
240 : // NOTE: ubrk_getRuleStatus() is NOT used for span classification. Apple's
241 : // libicucore does not include the UAX #29 rule-status tags in its compiled
242 : // word break rules, so the function returns 0 for all letter/number spans
243 : // and non-zero for certain whitespace sequences — the inverse of the
244 : // upstream ICU convention. Character-based classification is both more
245 : // portable and simpler.
246 6 : static final _hasWordChar = RegExp(r'[\p{L}\p{N}]', unicode: true);
247 :
248 : // Strips leading/trailing non-letter/non-digit characters from a span.
249 : // Some ICU builds (including Apple's) group adjacent punctuation into the
250 : // same span as the word (e.g. "Hello," rather than "Hello" + ",").
251 6 : static final _leadingNonWord = RegExp(r'^[^\p{L}\p{N}]+', unicode: true);
252 6 : static final _trailingNonWord = RegExp(r'[^\p{L}\p{N}]+$', unicode: true);
253 :
254 2 : @override
255 : List<String> tokenise(String text) =>
256 10 : tokeniseSpans(text).map((s) => s.text).toList(growable: false);
257 :
258 2 : @override
259 : List<TokenSpan> tokeniseSpans(String text) {
260 2 : if (text.isEmpty) return const [];
261 :
262 2 : final codeUnits = text.codeUnits;
263 2 : final len = codeUnits.length;
264 :
265 : // Allocate a native UTF-16 buffer and an error-code cell.
266 : final textBuf = calloc<Uint16>(len);
267 : final statusBuf = calloc<Int32>();
268 :
269 : try {
270 : // Copy the Dart string's UTF-16 code units into native memory.
271 : // Dart strings are UTF-16 encoded; codeUnits gives the correct uint16_t
272 : // values for all characters, including supplementary characters encoded
273 : // as surrogate pairs.
274 4 : for (var i = 0; i < len; i++) {
275 4 : textBuf[i] = codeUnits[i];
276 : }
277 :
278 : // Open a UBRK_WORD iterator. Passing address 0 for locale means
279 : // "default locale"; for script-level word breaking this is fine.
280 4 : final bi = _ubrkOpen(
281 : _ubrkWord,
282 2 : Pointer<Utf8>.fromAddress(0), // nullptr → default locale
283 : textBuf,
284 : len,
285 : statusBuf,
286 : );
287 :
288 4 : _checkStatus(statusBuf.value, 'ubrk_open');
289 :
290 : try {
291 2 : final spans = <TokenSpan>[];
292 : var start = 0;
293 :
294 : while (true) {
295 4 : final end = _ubrkNext(bi);
296 2 : if (end == _ubrkDone) break;
297 :
298 2 : final span = text.substring(start, end);
299 :
300 : // Include the span only if it contains at least one letter or digit.
301 : // Then strip any punctuation that was grouped at either end,
302 : // tracking exactly how many characters were trimmed from each side
303 : // so the reported offsets describe the trimmed word, not the raw
304 : // ICU span. `_leadingNonWord`/`_trailingNonWord` are anchored
305 : // (`^`/`$`), so `firstMatch` finds precisely what `replaceFirst`
306 : // would have removed.
307 4 : if (_hasWordChar.hasMatch(span)) {
308 6 : final leadingTrim = _leadingNonWord.firstMatch(span)?.end ?? 0;
309 4 : final trailingMatch = _trailingNonWord.firstMatch(span);
310 : final trailingTrim = trailingMatch == null
311 : ? 0
312 6 : : span.length - trailingMatch.start;
313 2 : final word = span.substring(
314 : leadingTrim,
315 4 : span.length - trailingTrim,
316 : );
317 2 : if (word.isNotEmpty) {
318 2 : spans.add(
319 6 : TokenSpan(word, start + leadingTrim, end - trailingTrim),
320 : );
321 : }
322 : }
323 :
324 : start = end;
325 : }
326 :
327 : return spans;
328 : } finally {
329 4 : _ubrkClose(bi);
330 : }
331 : } finally {
332 2 : calloc.free(textBuf);
333 2 : calloc.free(statusBuf);
334 : }
335 : }
336 :
337 : /// Throws [StateError] if [statusCode] indicates a fatal ICU error.
338 : ///
339 : /// ICU's UErrorCode convention (unicode/utypes.h):
340 : /// < 0 warnings (non-fatal — e.g. U_USING_DEFAULT_WARNING = -127)
341 : /// = 0 U_ZERO_ERROR (success)
342 : /// > 0 errors (fatal — e.g. U_ILLEGAL_ARGUMENT_ERROR = 1)
343 2 : static void _checkStatus(int statusCode, String fn) {
344 2 : if (statusCode > 0) {
345 0 : throw StateError('ICU error in $fn: UErrorCode $statusCode');
346 : }
347 : }
348 : }
|
